From c48d12ca6c5b9299f17ddff3469aaac651679a66 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 31 May 2023 15:45:05 +0200 Subject: [PATCH 001/100] Upgrade Kustomize to version 4.5.4 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 57e36fc93..e4a8cf020 100644 --- a/Makefile +++ b/Makefile @@ -157,7 +157,7 @@ ENVTEST ?= $(LOCALBIN)/setup-envtest OPERATOR_SDK ?= $(LOCALBIN)/operator-sdk ## Tool Versions -KUSTOMIZE_VERSION ?= v3.8.7 +KUSTOMIZE_VERSION ?= v4.5.4 CONTROLLER_TOOLS_VERSION ?= v0.9.2 OPERATOR_SDK_VERSION ?= v1.27.0 From 1616dedb7587ce137583487abd8846817b78d5bb Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 31 May 2023 15:46:48 +0200 Subject: [PATCH 002/100] Use Kustomize to bundle MCAD CRDs --- Makefile | 6 - .../crd/bases/mcad.ibm.com_appwrappers.yaml | 799 -- config/crd/bases/mcad.ibm.com_queuejobs.yaml | 6841 ----------------- .../bases/mcad.ibm.com_schedulingspecs.yaml | 68 - config/crd/kustomization.yaml | 4 +- config/crd/mcad/kustomization.yaml | 2 + 6 files changed, 3 insertions(+), 7717 deletions(-) delete mode 100644 config/crd/bases/mcad.ibm.com_appwrappers.yaml delete mode 100644 config/crd/bases/mcad.ibm.com_queuejobs.yaml delete mode 100644 config/crd/bases/mcad.ibm.com_schedulingspecs.yaml create mode 100644 config/crd/mcad/kustomization.yaml diff --git a/Makefile b/Makefile index e4a8cf020..182059677 100644 --- a/Makefile +++ b/Makefile @@ -248,9 +248,3 @@ catalog-push: ## Push a catalog image. .PHONY: test-unit test-unit: manifests generate fmt vet envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -coverprofile cover.out - -.PHONY: sync-crd -sync-crd: - curl -L -o bundle/manifests/mcad.ibm.com_appwrappers.yaml https://raw.githubusercontent.com/project-codeflare/multi-cluster-app-dispatcher/main/config/crd/bases/mcad.ibm.com_appwrappers.yaml - curl -L -o bundle/manifests/mcad.ibm.com_queuejobs.yaml https://raw.githubusercontent.com/project-codeflare/multi-cluster-app-dispatcher/main/config/crd/bases/mcad.ibm.com_queuejobs.yaml - curl -L -o bundle/manifests/mcad.ibm.com_schedulingspecs.yaml https://raw.githubusercontent.com/project-codeflare/multi-cluster-app-dispatcher/main/config/crd/bases/mcad.ibm.com_schedulingspecs.yaml diff --git a/config/crd/bases/mcad.ibm.com_appwrappers.yaml b/config/crd/bases/mcad.ibm.com_appwrappers.yaml deleted file mode 100644 index eb4327b4f..000000000 --- a/config/crd/bases/mcad.ibm.com_appwrappers.yaml +++ /dev/null @@ -1,799 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.4.1 - creationTimestamp: null - name: appwrappers.mcad.ibm.com -spec: - group: mcad.ibm.com - names: - kind: AppWrapper - listKind: AppWrapperList - plural: appwrappers - singular: appwrapper - scope: Namespaced - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: Definition of AppWrapper class - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: AppWrapperSpec describes how the App Wrapper will look like. - properties: - priority: - format: int32 - type: integer - priorityslope: - format: float - type: number - resources: - description: a collection of AppWrapperResource - properties: - GenericItems: - items: - description: AppWrapperResource is App Wrapper aggregation resource - properties: - allocated: - description: The number of allocated replicas from this - resource type - format: int32 - type: integer - apiVersion: - description: 'APIVersion defines the versioned schema of - this representation of an object. Servers should convert - recognized schemas to the latest internal value, and may - reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - completionstatus: - description: Optional field that drives completion status of appwrapper. - This field within an item of an appwrapper determines the full state of appwrapper. - The completionstatus field contains a list of conditions that make the associate item considered - completed, for instance :- completion conditions could be "Complete" or "Failed". - The associated item's level .status.conditions[].type field is monitored for any one of these conditions. Once all items with this - option is set and the conditionstatus is met the entire appwrapper state will be changed to one of the valid appwrapper completion state. Note :- this is an AND - operation for all items where this option is set. See the list of appwrapper states for a list of valid complete states. - type: string - custompodresources: - description: Optional section that specifies resource requirements - for non-standard k8s resources, follows same format as - that of standard k8s resources - items: - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: ResourceList is a set of (resource name, - quantity) pairs. - type: object - replicas: - type: integer - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'todo: replace with Containers []Container - Contain v1.ResourceRequirements' - type: object - required: - - replicas - - requests - type: object - type: array - generictemplate: - description: The template for the resource; it is now a - raw text because we don't know for what resource it should - be instantiated - type: object - x-kubernetes-embedded-resource: true - x-kubernetes-preserve-unknown-fields: true - kind: - description: 'Kind is a string value representing the REST - resource this object represents. Servers may infer this - from the endpoint the client submits requests to. Cannot - be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - minavailable: - description: The minimal available pods to run for this - AppWrapper; the default value is nil - format: int32 - type: integer - priority: - description: The priority of this resource - format: int32 - type: integer - priorityslope: - description: The increasing rate of priority value for this - resource - format: float - type: number - replicas: - description: Replicas is the number of desired replicas - format: int32 - type: integer - type: object - type: array - Items: - items: - description: 'AppWrapperResource is App Wrapper aggregation - resource todo: To be depricated' - properties: - allocatedreplicas: - description: The number of allocated replicas from this - resource type - format: int32 - type: integer - apiVersion: - description: 'APIVersion defines the versioned schema of - this representation of an object. Servers should convert - recognized schemas to the latest internal value, and may - reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST - resource this object represents. Servers may infer this - from the endpoint the client submits requests to. Cannot - be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - minavailable: - description: The minimal available pods to run for this - AppWrapper; the default value is nil - format: int32 - type: integer - priority: - description: The priority of this resource - format: int32 - type: integer - priorityslope: - description: The increasing rate of priority value for this - resource - format: float - type: number - replicas: - description: Replicas is the number of desired replicas - format: int32 - type: integer - template: - description: The template for the resource; it is now a - raw text because we don't know for what resource it should - be instantiated - type: object - x-kubernetes-preserve-unknown-fields: true - type: - description: The type of the resource (is the resource a - Pod, a ReplicaSet, a ... ?) - type: string - required: - - template - type: object - type: array - apiVersion: - description: 'APIVersion defines the versioned schema of this - representation of an object. Servers should convert recognized - schemas to the latest internal value, and may reject unrecognized - values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource - this object represents. Servers may infer this from the endpoint - the client submits requests to. Cannot be updated. In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - description: ListMeta describes metadata that synthetic resources - must have, including lists and various status objects. A resource - may have only one of {ObjectMeta, ListMeta}. - properties: - continue: - description: continue may be set if the user set a limit on - the number of items returned, and indicates that the server - has more data available. The value is opaque and may be - used to issue another request to the endpoint that served - this list to retrieve the next set of available objects. - Continuing a consistent list may not be possible if the - server configuration has changed or more than a few minutes - have passed. The resourceVersion field returned when using - this continue value will be identical to the value in the - first response, unless you have received this token from - an error message. - type: string - remainingItemCount: - description: remainingItemCount is the number of subsequent - items in the list which are not included in this list response. - If the list request contained label or field selectors, - then the number of remaining items is unknown and the field - will be left unset and omitted during serialization. If - the list is complete (either because it is not chunking - or because this is the last chunk), then there are no more - remaining items and this field will be left unset and omitted - during serialization. Servers older than v1.15 do not set - this field. The intended use of the remainingItemCount is - *estimating* the size of a collection. Clients should not - rely on the remainingItemCount to be set or to be exact. - format: int64 - type: integer - resourceVersion: - description: 'String that identifies the server''s internal - version of this object that can be used by clients to determine - when objects have changed. Value must be treated as opaque - by clients and passed unmodified back to the server. Populated - by the system. Read-only. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' - type: string - selfLink: - description: "selfLink is a URL representing this object. - Populated by the system. Read-only. \n DEPRECATED Kubernetes - will stop propagating this field in 1.20 release and the - field is planned to be removed in 1.21 release." - type: string - type: object - type: object - schedulingSpec: - description: SchedSpec specifies the parameters for scheduling. - properties: - minAvailable: - type: integer - requeuing: - description: Specification of the requeuing strategy based on - waiting time - properties: - initialTimeInSeconds: - type: integer - timeInSeconds: - type: integer - default: 300 - maxTimeInSeconds: - type: integer - default: 0 - growthType: - type: string - default: "exponential" - numRequeuings: - type: integer - default: 0 - maxNumRequeuings: - type: integer - default: 0 - type: object - nodeSelector: - additionalProperties: - type: string - type: object - type: object - selector: - description: A label selector is a label query over a set of resources. - The result of matchLabels and matchExpressions are ANDed. An empty - label selector matches all objects. A null label selector matches - no objects. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. - The requirements are ANDed. - items: - description: A label selector requirement is a selector that - contains values, a key, and an operator that relates the key - and values. - properties: - key: - description: key is the label key that the selector applies - to. - type: string - operator: - description: operator represents a key's relationship to - a set of values. Valid operators are In, NotIn, Exists - and DoesNotExist. - type: string - values: - description: values is an array of string values. If the - operator is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single - {key,value} in the matchLabels map is equivalent to an element - of matchExpressions, whose key field is "key", the operator - is "In", and the values array contains only "value". The requirements - are ANDed. - type: object - type: object - service: - description: AppWrapperService is App Wrapper service definition - properties: - spec: - description: ServiceSpec describes the attributes that a user - creates on a service. - properties: - allocateLoadBalancerNodePorts: - description: allocateLoadBalancerNodePorts defines if NodePorts - will be automatically allocated for services with type LoadBalancer. Default - is "true". It may be set to "false" if the cluster load-balancer - does not rely on NodePorts. allocateLoadBalancerNodePorts - may only be set for services with type LoadBalancer and - will be cleared if the type is changed to any other type. - This field is alpha-level and is only honored by servers - that enable the ServiceLBNodePortControl feature. - type: boolean - clusterIP: - description: 'clusterIP is the IP address of the service and - is usually assigned randomly. If an address is specified - manually, is in-range (as per system configuration), and - is not in use, it will be allocated to the service; otherwise - creation of the service will fail. This field may not be - changed through updates unless the type field is also being - changed to ExternalName (which requires this field to be - blank) or the type field is being changed from ExternalName - (in which case this field may optionally be specified, as - describe above). Valid values are "None", empty string - (""), or a valid IP address. Setting this to "None" makes - a "headless service" (no virtual IP), which is useful when - direct endpoint connections are preferred and proxying is - not required. Only applies to types ClusterIP, NodePort, - and LoadBalancer. If this field is specified when creating - a Service of type ExternalName, creation will fail. This - field will be wiped when updating a Service to type ExternalName. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' - type: string - clusterIPs: - description: "ClusterIPs is a list of IP addresses assigned - to this service, and are usually assigned randomly. If - an address is specified manually, is in-range (as per system - configuration), and is not in use, it will be allocated - to the service; otherwise creation of the service will fail. - This field may not be changed through updates unless the - type field is also being changed to ExternalName (which - requires this field to be empty) or the type field is being - changed from ExternalName (in which case this field may - optionally be specified, as describe above). Valid values - are \"None\", empty string (\"\"), or a valid IP address. - \ Setting this to \"None\" makes a \"headless service\" - (no virtual IP), which is useful when direct endpoint connections - are preferred and proxying is not required. Only applies - to types ClusterIP, NodePort, and LoadBalancer. If this - field is specified when creating a Service of type ExternalName, - creation will fail. This field will be wiped when updating - a Service to type ExternalName. If this field is not specified, - it will be initialized from the clusterIP field. If this - field is specified, clients must ensure that clusterIPs[0] - and clusterIP have the same value. \n Unless the \"IPv6DualStack\" - feature gate is enabled, this field is limited to one value, - which must be the same as the clusterIP field. If the feature - gate is enabled, this field may hold a maximum of two entries - (dual-stack IPs, in either order). These IPs must correspond - to the values of the ipFamilies field. Both clusterIPs and - ipFamilies are governed by the ipFamilyPolicy field. More - info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies" - items: - type: string - type: array - x-kubernetes-list-type: atomic - externalIPs: - description: externalIPs is a list of IP addresses for which - nodes in the cluster will also accept traffic for this service. These - IPs are not managed by Kubernetes. The user is responsible - for ensuring that traffic arrives at a node with this IP. A - common example is external load-balancers that are not part - of the Kubernetes system. - items: - type: string - type: array - externalName: - description: externalName is the external reference that discovery - mechanisms will return as an alias for this service (e.g. - a DNS CNAME record). No proxying will be involved. Must - be a lowercase RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) - and requires Type to be - type: string - externalTrafficPolicy: - description: externalTrafficPolicy denotes if this Service - desires to route external traffic to node-local or cluster-wide - endpoints. "Local" preserves the client source IP and avoids - a second hop for LoadBalancer and Nodeport type services, - but risks potentially imbalanced traffic spreading. "Cluster" - obscures the client source IP and may cause a second hop - to another node, but should have good overall load-spreading. - type: string - healthCheckNodePort: - description: healthCheckNodePort specifies the healthcheck - nodePort for the service. This only applies when type is - set to LoadBalancer and externalTrafficPolicy is set to - Local. If a value is specified, is in-range, and is not - in use, it will be used. If not specified, a value will - be automatically allocated. External systems (e.g. load-balancers) - can use this port to determine if a given node holds endpoints - for this service or not. If this field is specified when - creating a Service which does not need it, creation will - fail. This field will be wiped when updating a Service to - no longer need it (e.g. changing type). - format: int32 - type: integer - ipFamilies: - description: "IPFamilies is a list of IP families (e.g. IPv4, - IPv6) assigned to this service, and is gated by the \"IPv6DualStack\" - feature gate. This field is usually assigned automatically - based on cluster configuration and the ipFamilyPolicy field. - If this field is specified manually, the requested family - is available in the cluster, and ipFamilyPolicy allows it, - it will be used; otherwise creation of the service will - fail. This field is conditionally mutable: it allows for - adding or removing a secondary IP family, but it does not - allow changing the primary IP family of the Service. Valid - values are \"IPv4\" and \"IPv6\". This field only applies - to Services of types ClusterIP, NodePort, and LoadBalancer, - and does apply to \"headless\" services. This field will - be wiped when updating a Service to type ExternalName. \n - This field may hold a maximum of two entries (dual-stack - families, in either order). These families must correspond - to the values of the clusterIPs field, if specified. Both - clusterIPs and ipFamilies are governed by the ipFamilyPolicy - field." - items: - description: IPFamily represents the IP Family (IPv4 or - IPv6). This type is used to express the family of an IP - expressed by a type (e.g. service.spec.ipFamilies). - type: string - type: array - x-kubernetes-list-type: atomic - ipFamilyPolicy: - description: IPFamilyPolicy represents the dual-stack-ness - requested or required by this Service, and is gated by the - "IPv6DualStack" feature gate. If there is no value provided, - then this field will be set to SingleStack. Services can - be "SingleStack" (a single IP family), "PreferDualStack" - (two IP families on dual-stack configured clusters or a - single IP family on single-stack clusters), or "RequireDualStack" - (two IP families on dual-stack configured clusters, otherwise - fail). The ipFamilies and clusterIPs fields depend on the - value of this field. This field will be wiped when updating - a service to type ExternalName. - type: string - loadBalancerIP: - description: 'Only applies to Service Type: LoadBalancer LoadBalancer - will get created with the IP specified in this field. This - feature depends on whether the underlying cloud-provider - supports specifying the loadBalancerIP when a load balancer - is created. This field will be ignored if the cloud-provider - does not support the feature.' - type: string - loadBalancerSourceRanges: - description: 'If specified and supported by the platform, - this will restrict traffic through the cloud-provider load-balancer - will be restricted to the specified client IPs. This field - will be ignored if the cloud-provider does not support the - feature." More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/' - items: - type: string - type: array - ports: - description: 'The list of ports that are exposed by this service. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' - items: - description: ServicePort contains information on service's - port. - properties: - appProtocol: - description: The application protocol for this port. - This field follows standard Kubernetes label syntax. - Un-prefixed names are reserved for IANA standard service - names (as per RFC-6335 and http://www.iana.org/assignments/service-names). - Non-standard protocols should use prefixed names such - as mycompany.com/my-custom-protocol. This is a beta - field that is guarded by the ServiceAppProtocol feature - gate and enabled by default. - type: string - name: - description: The name of this port within the service. - This must be a DNS_LABEL. All ports within a ServiceSpec - must have unique names. When considering the endpoints - for a Service, this must match the 'name' field in - the EndpointPort. Optional if only one ServicePort - is defined on this service. - type: string - nodePort: - description: 'The port on each node on which this service - is exposed when type is NodePort or LoadBalancer. Usually - assigned by the system. If a value is specified, in-range, - and not in use it will be used, otherwise the operation - will fail. If not specified, a port will be allocated - if this Service requires one. If this field is specified - when creating a Service which does not need it, creation - will fail. This field will be wiped when updating - a Service to no longer need it (e.g. changing type - from NodePort to ClusterIP). More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport' - format: int32 - type: integer - port: - description: The port that will be exposed by this service. - format: int32 - type: integer - protocol: - default: TCP - description: The IP protocol for this port. Supports - "TCP", "UDP", and "SCTP". Default is TCP. - type: string - targetPort: - anyOf: - - type: integer - - type: string - description: 'Number or name of the port to access on - the pods targeted by the service. Number must be in - the range 1 to 65535. Name must be an IANA_SVC_NAME. - If this is a string, it will be looked up as a named - port in the target Pod''s container ports. If this - is not specified, the value of the ''port'' field - is used (an identity map). This field is ignored for - services with clusterIP=None, and should be omitted - or set equal to the ''port'' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service' - x-kubernetes-int-or-string: true - required: - - port - type: object - type: array - x-kubernetes-list-map-keys: - - port - - protocol - x-kubernetes-list-type: map - publishNotReadyAddresses: - description: publishNotReadyAddresses indicates that any agent - which deals with endpoints for this Service should disregard - any indications of ready/not-ready. The primary use case - for setting this field is for a StatefulSet's Headless Service - to propagate SRV DNS records for its Pods for the purpose - of peer discovery. The Kubernetes controllers that generate - Endpoints and EndpointSlice resources for Services interpret - this to mean that all endpoints are considered "ready" even - if the Pods themselves are not. Agents which consume only - Kubernetes generated endpoints through the Endpoints or - EndpointSlice resources can safely assume this behavior. - type: boolean - selector: - additionalProperties: - type: string - description: 'Route service traffic to pods with label keys - and values matching this selector. If empty or not present, - the service is assumed to have an external process managing - its endpoints, which Kubernetes will not modify. Only applies - to types ClusterIP, NodePort, and LoadBalancer. Ignored - if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/' - type: object - sessionAffinity: - description: 'Supports "ClientIP" and "None". Used to maintain - session affinity. Enable client IP based session affinity. - Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' - type: string - sessionAffinityConfig: - description: sessionAffinityConfig contains the configurations - of session affinity. - properties: - clientIP: - description: clientIP contains the configurations of Client - IP based session affinity. - properties: - timeoutSeconds: - description: timeoutSeconds specifies the seconds - of ClientIP type session sticky time. The value - must be >0 && <=86400(for 1 day) if ServiceAffinity - == "ClientIP". Default value is 10800(for 3 hours). - format: int32 - type: integer - type: object - type: object - topologyKeys: - description: topologyKeys is a preference-order list of topology - keys which implementations of services should use to preferentially - sort endpoints when accessing this Service, it can not be - used at the same time as externalTrafficPolicy=Local. Topology - keys must be valid label keys and at most 16 keys may be - specified. Endpoints are chosen based on the first topology - key with available backends. If this field is specified - and all entries have no backends that match the topology - of the client, the service has no backends for that client - and connections should fail. The special value "*" may be - used to mean "any topology". This catch-all value, if used, - only makes sense as the last value in the list. If this - is not specified or empty, no topology constraints will - be applied. This field is alpha-level and is only honored - by servers that enable the ServiceTopology feature. - items: - type: string - type: array - type: - description: 'type determines how the Service is exposed. - Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, - NodePort, and LoadBalancer. "ClusterIP" allocates a cluster-internal - IP address for load-balancing to endpoints. Endpoints are - determined by the selector or if that is not specified, - by manual construction of an Endpoints object or EndpointSlice - objects. If clusterIP is "None", no virtual IP is allocated - and the endpoints are published as a set of endpoints rather - than a virtual IP. "NodePort" builds on ClusterIP and allocates - a port on every node which routes to the same endpoints - as the clusterIP. "LoadBalancer" builds on NodePort and - creates an external load-balancer (if supported in the current - cloud) which routes to the same endpoints as the clusterIP. - "ExternalName" aliases this service to the specified externalName. - Several other fields do not apply to ExternalName services. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types' - type: string - type: object - required: - - spec - type: object - required: - - resources - type: object - status: - description: AppWrapperStatus represents the current state of a AppWrapper - properties: - Succeeded: - description: The number of resources which reached phase Succeeded. - format: int32 - type: integer - canrun: - description: Can run? - type: boolean - conditions: - description: Represents the latest available observations of a appwrapper's - current condition. - items: - description: DeploymentCondition describes the state of a deployment - at a certain point. - properties: - lastTransitionMicroTime: - description: Last time the condition transitioned from one status - to another. - format: date-time - type: string - lastUpdateMicroTime: - description: The last time this condition was updated. - format: date-time - type: string - message: - description: A human readable message indicating details about - the transition. - type: string - reason: - description: The reason for the condition's last transition. - type: string - status: - description: Status of the condition, one of True, False, Unknown. - type: string - type: - description: Type of appwrapper condition. - type: string - required: - - status - - type - type: object - type: array - pendingpodconditions: - description: Represent conditions of pod(s) that failed scheduling. - items: - description: Describes scheduling failed condition(s) of a pod - properties: - podname: - description: Name of the pod - type: string - conditions: - description: Failed condition(s) of a pod - items: - description: DeploymentCondition describes the state of a deployment - at a certain point. - properties: - LastProbeTime: - description: Last time the condition transitioned from one status - to another. - format: date-time - type: string - LastTransitionTime: - description: The last time this condition was updated. - format: date-time - type: string - message: - description: A human readable message indicating details about - the transition. - type: string - reason: - description: The reason for the condition's last transition. - type: string - status: - description: Status of the condition, one of True, False, Unknown. - type: string - type: - description: Type of appwrapper condition. - type: string - required: - - status - - type - type: object - type: array - type: object - type: array - controllerfirsttimestamp: - description: Microsecond level timestamp when controller first sees - QueueJob (by Informer) - format: date-time - type: string - failed: - description: The number of resources which reached phase Failed. - format: int32 - type: integer - filterignore: - description: Tell Informer to ignore this update message (do not generate - a controller event) - type: boolean - isdispatched: - description: Is Dispatched? - type: boolean - local: - description: Indicate if message is a duplicate (for Informer to recognize - duplicate messages) - type: boolean - message: - type: string - pending: - description: The number of pending pods. - format: int32 - type: integer - queuejobstate: - description: State of QueueJob - Init, Queueing, HeadOfLine, Rejoining, - ... - type: string - running: - format: int32 - type: integer - sender: - description: Indicate sender of this message (extremely useful for - debugging) - type: string - state: - description: State - Pending, Running, Failed, Deleted - type: string - systempriority: - description: System defined Priority - format: float - type: number - template: - description: The minimal available resources to run for this AppWrapper - (is this different from the MinAvailable from JobStatus) - format: int32 - type: integer - type: object - required: - - spec - type: object - served: true - storage: true diff --git a/config/crd/bases/mcad.ibm.com_queuejobs.yaml b/config/crd/bases/mcad.ibm.com_queuejobs.yaml deleted file mode 100644 index b95b97743..000000000 --- a/config/crd/bases/mcad.ibm.com_queuejobs.yaml +++ /dev/null @@ -1,6841 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.4.1 - creationTimestamp: null - name: queuejobs.mcad.ibm.com -spec: - group: mcad.ibm.com - names: - kind: QueueJob - listKind: QueueJobList - plural: queuejobs - singular: queuejob - scope: Namespaced - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: Specification of the desired behavior of a cron job, including - the minAvailable and the requeuing strategy - properties: - schedulerName: - type: string - schedulingSpec: - description: SchedSpec specifies the parameters for scheduling. - properties: - minAvailable: - type: integer - requeuing: - description: Specification of the requeuing strategy based on - waiting time - properties: - initialTimeInSeconds: - type: integer - timeInSeconds: - type: integer - default: 300 - maxTimeInSeconds: - type: integer - default: 0 - growthType: - type: string - default: "exponential" - numRequeuings: - type: integer - default: 0 - maxNumRequeuings: - type: integer - default: 0 - type: object - nodeSelector: - additionalProperties: - type: string - type: object - type: object - taskSpecs: - description: TaskSpecs specifies the task specification of QueueJob - items: - description: TaskSpec specifies the task specification of QueueJob - properties: - replicas: - description: Replicas specifies the replicas of this TaskSpec - in QueueJob. - format: int32 - type: integer - selector: - description: A label query over pods that should match the pod - count. Normally, the system sets this field for you. - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: A label selector requirement is a selector - that contains values, a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: operator represents a key's relationship - to a set of values. Valid operators are In, NotIn, - Exists and DoesNotExist. - type: string - values: - description: values is an array of string values. - If the operator is In or NotIn, the values array - must be non-empty. If the operator is Exists or - DoesNotExist, the values array must be empty. This - array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. - A single {key,value} in the matchLabels map is equivalent - to an element of matchExpressions, whose key field is - "key", the operator is "In", and the values array contains - only "value". The requirements are ANDed. - type: object - type: object - template: - description: Specifies the pod that will be created for this - TaskSpec when executing a QueueJob - properties: - metadata: - description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata' - type: object - spec: - description: 'Specification of the desired behavior of the - pod. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' - properties: - activeDeadlineSeconds: - description: Optional duration in seconds the pod may - be active on the node relative to StartTime before - the system will actively try to mark it failed and - kill associated containers. Value must be a positive - integer. - format: int64 - type: integer - affinity: - description: If specified, the pod's scheduling constraints - properties: - nodeAffinity: - description: Describes node affinity scheduling - rules for the pod. - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule - pods to nodes that satisfy the affinity expressions - specified by this field, but it may choose - a node that violates one or more of the expressions. - The node that is most preferred is the one - with the greatest sum of weights, i.e. for - each node that meets all of the scheduling - requirements (resource request, requiredDuringScheduling - affinity expressions, etc.), compute a sum - by iterating through the elements of this - field and adding "weight" to the sum if the - node matches the corresponding matchExpressions; - the node(s) with the highest sum are the most - preferred. - items: - description: An empty preferred scheduling - term matches all objects with implicit weight - 0 (i.e. it's a no-op). A null preferred - scheduling term matches no objects (i.e. - is also a no-op). - properties: - preference: - description: A node selector term, associated - with the corresponding weight. - properties: - matchExpressions: - description: A list of node selector - requirements by node's labels. - items: - description: A node selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: The label key that - the selector applies to. - type: string - operator: - description: Represents a key's - relationship to a set of values. - Valid operators are In, NotIn, - Exists, DoesNotExist. Gt, - and Lt. - type: string - values: - description: An array of string - values. If the operator is - In or NotIn, the values array - must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. - If the operator is Gt or Lt, - the values array must have - a single element, which will - be interpreted as an integer. - This array is replaced during - a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector - requirements by node's fields. - items: - description: A node selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: The label key that - the selector applies to. - type: string - operator: - description: Represents a key's - relationship to a set of values. - Valid operators are In, NotIn, - Exists, DoesNotExist. Gt, - and Lt. - type: string - values: - description: An array of string - values. If the operator is - In or NotIn, the values array - must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. - If the operator is Gt or Lt, - the values array must have - a single element, which will - be interpreted as an integer. - This array is replaced during - a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - weight: - description: Weight associated with matching - the corresponding nodeSelectorTerm, - in the range 1-100. - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified - by this field are not met at scheduling time, - the pod will not be scheduled onto the node. - If the affinity requirements specified by - this field cease to be met at some point during - pod execution (e.g. due to an update), the - system may or may not try to eventually evict - the pod from its node. - properties: - nodeSelectorTerms: - description: Required. A list of node selector - terms. The terms are ORed. - items: - description: A null or empty node selector - term matches no objects. The requirements - of them are ANDed. The TopologySelectorTerm - type implements a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector - requirements by node's labels. - items: - description: A node selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: The label key that - the selector applies to. - type: string - operator: - description: Represents a key's - relationship to a set of values. - Valid operators are In, NotIn, - Exists, DoesNotExist. Gt, - and Lt. - type: string - values: - description: An array of string - values. If the operator is - In or NotIn, the values array - must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. - If the operator is Gt or Lt, - the values array must have - a single element, which will - be interpreted as an integer. - This array is replaced during - a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector - requirements by node's fields. - items: - description: A node selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: The label key that - the selector applies to. - type: string - operator: - description: Represents a key's - relationship to a set of values. - Valid operators are In, NotIn, - Exists, DoesNotExist. Gt, - and Lt. - type: string - values: - description: An array of string - values. If the operator is - In or NotIn, the values array - must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. - If the operator is Gt or Lt, - the values array must have - a single element, which will - be interpreted as an integer. - This array is replaced during - a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - description: Describes pod affinity scheduling rules - (e.g. co-locate this pod in the same node, zone, - etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule - pods to nodes that satisfy the affinity expressions - specified by this field, but it may choose - a node that violates one or more of the expressions. - The node that is most preferred is the one - with the greatest sum of weights, i.e. for - each node that meets all of the scheduling - requirements (resource request, requiredDuringScheduling - affinity expressions, etc.), compute a sum - by iterating through the elements of this - field and adding "weight" to the sum if the - node has pods which matches the corresponding - podAffinityTerm; the node(s) with the highest - sum are the most preferred. - items: - description: The weights of all of the matched - WeightedPodAffinityTerm fields are added - per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity - term, associated with the corresponding - weight. - properties: - labelSelector: - description: A label query over a - set of resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions - is a list of label selector - requirements. The requirements - are ANDed. - items: - description: A label selector - requirement is a selector - that contains values, a key, - and an operator that relates - the key and values. - properties: - key: - description: key is the - label key that the selector - applies to. - type: string - operator: - description: operator represents - a key's relationship to - a set of values. Valid - operators are In, NotIn, - Exists and DoesNotExist. - type: string - values: - description: values is an - array of string values. - If the operator is In - or NotIn, the values array - must be non-empty. If - the operator is Exists - or DoesNotExist, the values - array must be empty. This - array is replaced during - a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a - map of {key,value} pairs. A - single {key,value} in the matchLabels - map is equivalent to an element - of matchExpressions, whose key - field is "key", the operator - is "In", and the values array - contains only "value". The requirements - are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies - which namespaces the labelSelector - applies to (matches against); null - or empty list means "this pod's - namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located - (affinity) or not co-located (anti-affinity) - with the pods matching the labelSelector - in the specified namespaces, where - co-located is defined as running - on a node whose value of the label - with key topologyKey matches that - of any node on which any of the - selected pods is running. Empty - topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching - the corresponding podAffinityTerm, in - the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified - by this field are not met at scheduling time, - the pod will not be scheduled onto the node. - If the affinity requirements specified by - this field cease to be met at some point during - pod execution (e.g. due to a pod label update), - the system may or may not try to eventually - evict the pod from its node. When there are - multiple elements, the lists of nodes corresponding - to each podAffinityTerm are intersected, i.e. - all terms must be satisfied. - items: - description: Defines a set of pods (namely - those matching the labelSelector relative - to the given namespace(s)) that this pod - should be co-located (affinity) or not co-located - (anti-affinity) with, where co-located is - defined as running on a node whose value - of the label with key matches - that of any node on which a pod of the set - of pods is running - properties: - labelSelector: - description: A label query over a set - of resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions is a - list of label selector requirements. - The requirements are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: operator represents - a key's relationship to a - set of values. Valid operators - are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an array - of string values. If the operator - is In or NotIn, the values - array must be non-empty. If - the operator is Exists or - DoesNotExist, the values array - must be empty. This array - is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map - of {key,value} pairs. A single {key,value} - in the matchLabels map is equivalent - to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are - ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which - namespaces the labelSelector applies - to (matches against); null or empty - list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located - (affinity) or not co-located (anti-affinity) - with the pods matching the labelSelector - in the specified namespaces, where co-located - is defined as running on a node whose - value of the label with key topologyKey - matches that of any node on which any - of the selected pods is running. Empty - topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - description: Describes pod anti-affinity scheduling - rules (e.g. avoid putting this pod in the same - node, zone, etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule - pods to nodes that satisfy the anti-affinity - expressions specified by this field, but it - may choose a node that violates one or more - of the expressions. The node that is most - preferred is the one with the greatest sum - of weights, i.e. for each node that meets - all of the scheduling requirements (resource - request, requiredDuringScheduling anti-affinity - expressions, etc.), compute a sum by iterating - through the elements of this field and adding - "weight" to the sum if the node has pods which - matches the corresponding podAffinityTerm; - the node(s) with the highest sum are the most - preferred. - items: - description: The weights of all of the matched - WeightedPodAffinityTerm fields are added - per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity - term, associated with the corresponding - weight. - properties: - labelSelector: - description: A label query over a - set of resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions - is a list of label selector - requirements. The requirements - are ANDed. - items: - description: A label selector - requirement is a selector - that contains values, a key, - and an operator that relates - the key and values. - properties: - key: - description: key is the - label key that the selector - applies to. - type: string - operator: - description: operator represents - a key's relationship to - a set of values. Valid - operators are In, NotIn, - Exists and DoesNotExist. - type: string - values: - description: values is an - array of string values. - If the operator is In - or NotIn, the values array - must be non-empty. If - the operator is Exists - or DoesNotExist, the values - array must be empty. This - array is replaced during - a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a - map of {key,value} pairs. A - single {key,value} in the matchLabels - map is equivalent to an element - of matchExpressions, whose key - field is "key", the operator - is "In", and the values array - contains only "value". The requirements - are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies - which namespaces the labelSelector - applies to (matches against); null - or empty list means "this pod's - namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located - (affinity) or not co-located (anti-affinity) - with the pods matching the labelSelector - in the specified namespaces, where - co-located is defined as running - on a node whose value of the label - with key topologyKey matches that - of any node on which any of the - selected pods is running. Empty - topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching - the corresponding podAffinityTerm, in - the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the anti-affinity requirements - specified by this field are not met at scheduling - time, the pod will not be scheduled onto the - node. If the anti-affinity requirements specified - by this field cease to be met at some point - during pod execution (e.g. due to a pod label - update), the system may or may not try to - eventually evict the pod from its node. When - there are multiple elements, the lists of - nodes corresponding to each podAffinityTerm - are intersected, i.e. all terms must be satisfied. - items: - description: Defines a set of pods (namely - those matching the labelSelector relative - to the given namespace(s)) that this pod - should be co-located (affinity) or not co-located - (anti-affinity) with, where co-located is - defined as running on a node whose value - of the label with key matches - that of any node on which a pod of the set - of pods is running - properties: - labelSelector: - description: A label query over a set - of resources, in this case pods. - properties: - matchExpressions: - description: matchExpressions is a - list of label selector requirements. - The requirements are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: operator represents - a key's relationship to a - set of values. Valid operators - are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an array - of string values. If the operator - is In or NotIn, the values - array must be non-empty. If - the operator is Exists or - DoesNotExist, the values array - must be empty. This array - is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map - of {key,value} pairs. A single {key,value} - in the matchLabels map is equivalent - to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are - ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which - namespaces the labelSelector applies - to (matches against); null or empty - list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located - (affinity) or not co-located (anti-affinity) - with the pods matching the labelSelector - in the specified namespaces, where co-located - is defined as running on a node whose - value of the label with key topologyKey - matches that of any node on which any - of the selected pods is running. Empty - topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - automountServiceAccountToken: - description: AutomountServiceAccountToken indicates - whether a service account token should be automatically - mounted. - type: boolean - containers: - description: List of containers belonging to the pod. - Containers cannot currently be added or removed. There - must be at least one container in a Pod. Cannot be - updated. - items: - description: A single application container that you - want to run within a pod. - properties: - args: - description: 'Arguments to the entrypoint. The - docker image''s CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded - using the container''s environment. If a variable - cannot be resolved, the reference in the input - string will be unchanged. The $(VAR_NAME) syntax - can be escaped with a double $$, ie: $$(VAR_NAME). - Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot - be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' - items: - type: string - type: array - command: - description: 'Entrypoint array. Not executed within - a shell. The docker image''s ENTRYPOINT is used - if this is not provided. Variable references - $(VAR_NAME) are expanded using the container''s - environment. If a variable cannot be resolved, - the reference in the input string will be unchanged. - The $(VAR_NAME) syntax can be escaped with a - double $$, ie: $$(VAR_NAME). Escaped references - will never be expanded, regardless of whether - the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' - items: - type: string - type: array - env: - description: List of environment variables to - set in the container. Cannot be updated. - items: - description: EnvVar represents an environment - variable present in a Container. - properties: - name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. - type: string - value: - description: 'Variable references $(VAR_NAME) - are expanded using the previous defined - environment variables in the container - and any service environment variables. - If a variable cannot be resolved, the - reference in the input string will be - unchanged. The $(VAR_NAME) syntax can - be escaped with a double $$, ie: $$(VAR_NAME). - Escaped references will never be expanded, - regardless of whether the variable exists - or not. Defaults to "".' - type: string - valueFrom: - description: Source for the environment - variable's value. Cannot be used if value - is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - description: 'Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. - apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the - ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - fieldRef: - description: 'Selects a field of the - pod: supports metadata.name, metadata.namespace, - `metadata.labels['''']`, `metadata.annotations['''']`, - spec.nodeName, spec.serviceAccountName, - status.hostIP, status.podIP, status.podIPs.' - properties: - apiVersion: - description: Version of the schema - the FieldPath is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to - select in the specified API version. - type: string - required: - - fieldPath - type: object - resourceFieldRef: - description: 'Selects a resource of - the container: only resources limits - and requests (limits.cpu, limits.memory, - limits.ephemeral-storage, requests.cpu, - requests.memory and requests.ephemeral-storage) - are currently supported.' - properties: - containerName: - description: 'Container name: required - for volumes, optional for env - vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource - to select' - type: string - required: - - resource - type: object - secretKeyRef: - description: Selects a key of a secret - in the pod's namespace - properties: - key: - description: The key of the secret - to select from. Must be a valid - secret key. - type: string - name: - description: 'Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. - apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the - Secret or its key must be defined - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - description: List of sources to populate environment - variables in the container. The keys defined - within a source must be a C_IDENTIFIER. All - invalid keys will be reported as an event when - the container is starting. When a key exists - in multiple sources, the value associated with - the last source will take precedence. Values - defined by an Env with a duplicate key will - take precedence. Cannot be updated. - items: - description: EnvFromSource represents the source - of a set of ConfigMaps - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - description: 'Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - optional: - description: Specify whether the ConfigMap - must be defined - type: boolean - type: object - prefix: - description: An optional identifier to prepend - to each key in the ConfigMap. Must be - a C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from - properties: - name: - description: 'Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - optional: - description: Specify whether the Secret - must be defined - type: boolean - type: object - type: object - type: array - image: - description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level - config management to default or override container - images in workload controllers like Deployments - and StatefulSets.' - type: string - imagePullPolicy: - description: 'Image pull policy. One of Always, - Never, IfNotPresent. Defaults to Always if :latest - tag is specified, or IfNotPresent otherwise. - Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' - type: string - lifecycle: - description: Actions that the management system - should take in response to container lifecycle - events. Cannot be updated. - properties: - postStart: - description: 'PostStart is called immediately - after a container is created. If the handler - fails, the container is terminated and restarted - according to its restart policy. Other management - of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' - properties: - exec: - description: One and only one of the following - should be specified. Exec specifies - the action to take. - properties: - command: - description: Command is the command - line to execute inside the container, - the working directory for the command is - root ('/') in the container's filesystem. - The command is simply exec'd, it - is not run inside a shell, so traditional - shell instructions ('|', etc) won't - work. To use a shell, you need to - explicitly call out to that shell. - Exit status of 0 is treated as live/healthy - and non-zero is unhealthy. - items: - type: string - type: array - type: object - httpGet: - description: HTTPGet specifies the http - request to perform. - properties: - host: - description: Host name to connect - to, defaults to the pod IP. You - probably want to set "Host" in httpHeaders - instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: The header field - name - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the - port to access on the container. - Number must be in the range 1 to - 65535. Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - tcpSocket: - description: 'TCPSocket specifies an action - involving a TCP port. TCP hooks not - yet supported TODO: implement a realistic - TCP lifecycle hook' - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the - port to access on the container. - Number must be in the range 1 to - 65535. Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: 'PreStop is called immediately - before a container is terminated due to - an API request or management event such - as liveness/startup probe failure, preemption, - resource contention, etc. The handler is - not called if the container crashes or exits. - The reason for termination is passed to - the handler. The Pod''s termination grace - period countdown begins before the PreStop - hooked is executed. Regardless of the outcome - of the handler, the container will eventually - terminate within the Pod''s termination - grace period. Other management of the container - blocks until the hook completes or until - the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' - properties: - exec: - description: One and only one of the following - should be specified. Exec specifies - the action to take. - properties: - command: - description: Command is the command - line to execute inside the container, - the working directory for the command is - root ('/') in the container's filesystem. - The command is simply exec'd, it - is not run inside a shell, so traditional - shell instructions ('|', etc) won't - work. To use a shell, you need to - explicitly call out to that shell. - Exit status of 0 is treated as live/healthy - and non-zero is unhealthy. - items: - type: string - type: array - type: object - httpGet: - description: HTTPGet specifies the http - request to perform. - properties: - host: - description: Host name to connect - to, defaults to the pod IP. You - probably want to set "Host" in httpHeaders - instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: The header field - name - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the - port to access on the container. - Number must be in the range 1 to - 65535. Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - tcpSocket: - description: 'TCPSocket specifies an action - involving a TCP port. TCP hooks not - yet supported TODO: implement a realistic - TCP lifecycle hook' - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the - port to access on the container. - Number must be in the range 1 to - 65535. Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - description: 'Periodic probe of container liveness. - Container will be restarted if the probe fails. - Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - properties: - exec: - description: One and only one of the following - should be specified. Exec specifies the - action to take. - properties: - command: - description: Command is the command line - to execute inside the container, the - working directory for the command is - root ('/') in the container's filesystem. - The command is simply exec'd, it is - not run inside a shell, so traditional - shell instructions ('|', etc) won't - work. To use a shell, you need to explicitly - call out to that shell. Exit status - of 0 is treated as live/healthy and - non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: Minimum consecutive failures - for the probe to be considered failed after - having succeeded. Defaults to 3. Minimum - value is 1. - format: int32 - type: integer - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: Host name to connect to, - defaults to the pod IP. You probably - want to set "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: The header field name - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: 'Number of seconds after the - container has started before liveness probes - are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - periodSeconds: - description: How often (in seconds) to perform - the probe. Default to 10 seconds. Minimum - value is 1. - format: int32 - type: integer - successThreshold: - description: Minimum consecutive successes - for the probe to be considered successful - after having failed. Defaults to 1. Must - be 1 for liveness and startup. Minimum value - is 1. - format: int32 - type: integer - tcpSocket: - description: 'TCPSocket specifies an action - involving a TCP port. TCP hooks not yet - supported TODO: implement a realistic TCP - lifecycle hook' - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - description: 'Number of seconds after which - the probe times out. Defaults to 1 second. - Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - type: object - name: - description: Name of the container specified as - a DNS_LABEL. Each container in a pod must have - a unique name (DNS_LABEL). Cannot be updated. - type: string - ports: - description: List of ports to expose from the - container. Exposing a port here gives the system - additional information about the network connections - a container uses, but is primarily informational. - Not specifying a port here DOES NOT prevent - that port from being exposed. Any port which - is listening on the default "0.0.0.0" address - inside a container will be accessible from the - network. Cannot be updated. - items: - description: ContainerPort represents a network - port in a single container. - properties: - containerPort: - description: Number of port to expose on - the pod's IP address. This must be a valid - port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external - port to. - type: string - hostPort: - description: Number of port to expose on - the host. If specified, this must be a - valid port number, 0 < x < 65536. If HostNetwork - is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: If specified, this must be - an IANA_SVC_NAME and unique within the - pod. Each named port in a pod must have - a unique name. Name for the port that - can be referred to by services. - type: string - protocol: - default: TCP - description: Protocol for port. Must be - UDP, TCP, or SCTP. Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: 'Periodic probe of container service - readiness. Container will be removed from service - endpoints if the probe fails. Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - properties: - exec: - description: One and only one of the following - should be specified. Exec specifies the - action to take. - properties: - command: - description: Command is the command line - to execute inside the container, the - working directory for the command is - root ('/') in the container's filesystem. - The command is simply exec'd, it is - not run inside a shell, so traditional - shell instructions ('|', etc) won't - work. To use a shell, you need to explicitly - call out to that shell. Exit status - of 0 is treated as live/healthy and - non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: Minimum consecutive failures - for the probe to be considered failed after - having succeeded. Defaults to 3. Minimum - value is 1. - format: int32 - type: integer - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: Host name to connect to, - defaults to the pod IP. You probably - want to set "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: The header field name - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: 'Number of seconds after the - container has started before liveness probes - are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - periodSeconds: - description: How often (in seconds) to perform - the probe. Default to 10 seconds. Minimum - value is 1. - format: int32 - type: integer - successThreshold: - description: Minimum consecutive successes - for the probe to be considered successful - after having failed. Defaults to 1. Must - be 1 for liveness and startup. Minimum value - is 1. - format: int32 - type: integer - tcpSocket: - description: 'TCPSocket specifies an action - involving a TCP port. TCP hooks not yet - supported TODO: implement a realistic TCP - lifecycle hook' - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - description: 'Number of seconds after which - the probe times out. Defaults to 1 second. - Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - type: object - resources: - description: 'Compute Resources required by this - container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Limits describes the maximum - amount of compute resources allowed. More - info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes the minimum - amount of compute resources required. If - Requests is omitted for a container, it - defaults to Limits if that is explicitly - specified, otherwise to an implementation-defined - value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' - type: object - type: object - securityContext: - description: 'Security options the pod should - run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ - More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' - properties: - allowPrivilegeEscalation: - description: 'AllowPrivilegeEscalation controls - whether a process can gain more privileges - than its parent process. This bool directly - controls if the no_new_privs flag will be - set on the container process. AllowPrivilegeEscalation - is true always when the container is: 1) - run as Privileged 2) has CAP_SYS_ADMIN' - type: boolean - capabilities: - description: The capabilities to add/drop - when running containers. Defaults to the - default set of capabilities granted by the - container runtime. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - drop: - description: Removed capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - type: object - privileged: - description: Run container in privileged mode. - Processes in privileged containers are essentially - equivalent to root on the host. Defaults - to false. - type: boolean - procMount: - description: procMount denotes the type of - proc mount to use for the containers. The - default is DefaultProcMount which uses the - container runtime defaults for readonly - paths and masked paths. This requires the - ProcMountType feature flag to be enabled. - type: string - readOnlyRootFilesystem: - description: Whether this container has a - read-only root filesystem. Default is false. - type: boolean - runAsGroup: - description: The GID to run the entrypoint - of the container process. Uses runtime default - if unset. May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes - precedence. - format: int64 - type: integer - runAsNonRoot: - description: Indicates that the container - must run as a non-root user. If true, the - Kubelet will validate the image at runtime - to ensure that it does not run as UID 0 - (root) and fail to start the container if - it does. If unset or false, no such validation - will be performed. May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes - precedence. - type: boolean - runAsUser: - description: The UID to run the entrypoint - of the container process. Defaults to user - specified in image metadata if unspecified. - May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes - precedence. - format: int64 - type: integer - seLinuxOptions: - description: The SELinux context to be applied - to the container. If unspecified, the container - runtime will allocate a random SELinux context - for each container. May also be set in - PodSecurityContext. If set in both SecurityContext - and PodSecurityContext, the value specified - in SecurityContext takes precedence. - properties: - level: - description: Level is SELinux level label - that applies to the container. - type: string - role: - description: Role is a SELinux role label - that applies to the container. - type: string - type: - description: Type is a SELinux type label - that applies to the container. - type: string - user: - description: User is a SELinux user label - that applies to the container. - type: string - type: object - seccompProfile: - description: The seccomp options to use by - this container. If seccomp options are provided - at both the pod & container level, the container - options override the pod options. - properties: - localhostProfile: - description: localhostProfile indicates - a profile defined in a file on the node - should be used. The profile must be - preconfigured on the node to work. Must - be a descending path, relative to the - kubelet's configured seccomp profile - location. Must only be set if type is - "Localhost". - type: string - type: - description: "type indicates which kind - of seccomp profile will be applied. - Valid options are: \n Localhost - a - profile defined in a file on the node - should be used. RuntimeDefault - the - container runtime default profile should - be used. Unconfined - no profile should - be applied." - type: string - required: - - type - type: object - windowsOptions: - description: The Windows specific settings - applied to all containers. If unspecified, - the options from the PodSecurityContext - will be used. If set in both SecurityContext - and PodSecurityContext, the value specified - in SecurityContext takes precedence. - properties: - gmsaCredentialSpec: - description: GMSACredentialSpec is where - the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) - inlines the contents of the GMSA credential - spec named by the GMSACredentialSpecName - field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is - the name of the GMSA credential spec - to use. - type: string - runAsUserName: - description: The UserName in Windows to - run the entrypoint of the container - process. Defaults to the user specified - in image metadata if unspecified. May - also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext - takes precedence. - type: string - type: object - type: object - startupProbe: - description: 'StartupProbe indicates that the - Pod has successfully initialized. If specified, - no other probes are executed until this completes - successfully. If this probe fails, the Pod will - be restarted, just as if the livenessProbe failed. - This can be used to provide different probe - parameters at the beginning of a Pod''s lifecycle, - when it might take a long time to load data - or warm a cache, than during steady-state operation. - This cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - properties: - exec: - description: One and only one of the following - should be specified. Exec specifies the - action to take. - properties: - command: - description: Command is the command line - to execute inside the container, the - working directory for the command is - root ('/') in the container's filesystem. - The command is simply exec'd, it is - not run inside a shell, so traditional - shell instructions ('|', etc) won't - work. To use a shell, you need to explicitly - call out to that shell. Exit status - of 0 is treated as live/healthy and - non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: Minimum consecutive failures - for the probe to be considered failed after - having succeeded. Defaults to 3. Minimum - value is 1. - format: int32 - type: integer - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: Host name to connect to, - defaults to the pod IP. You probably - want to set "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: The header field name - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: 'Number of seconds after the - container has started before liveness probes - are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - periodSeconds: - description: How often (in seconds) to perform - the probe. Default to 10 seconds. Minimum - value is 1. - format: int32 - type: integer - successThreshold: - description: Minimum consecutive successes - for the probe to be considered successful - after having failed. Defaults to 1. Must - be 1 for liveness and startup. Minimum value - is 1. - format: int32 - type: integer - tcpSocket: - description: 'TCPSocket specifies an action - involving a TCP port. TCP hooks not yet - supported TODO: implement a realistic TCP - lifecycle hook' - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - description: 'Number of seconds after which - the probe times out. Defaults to 1 second. - Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - type: object - stdin: - description: Whether this container should allocate - a buffer for stdin in the container runtime. - If this is not set, reads from stdin in the - container will always result in EOF. Default - is false. - type: boolean - stdinOnce: - description: Whether the container runtime should - close the stdin channel after it has been opened - by a single attach. When stdin is true the stdin - stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin - is opened on container start, is empty until - the first client attaches to stdin, and then - remains open and accepts data until the client - disconnects, at which time stdin is closed and - remains closed until the container is restarted. - If this flag is false, a container processes - that reads from stdin will never receive an - EOF. Default is false - type: boolean - terminationMessagePath: - description: 'Optional: Path at which the file - to which the container''s termination message - will be written is mounted into the container''s - filesystem. Message written is intended to be - brief final status, such as an assertion failure - message. Will be truncated by the node if greater - than 4096 bytes. The total message length across - all containers will be limited to 12kb. Defaults - to /dev/termination-log. Cannot be updated.' - type: string - terminationMessagePolicy: - description: Indicate how the termination message - should be populated. File will use the contents - of terminationMessagePath to populate the container - status message on both success and failure. - FallbackToLogsOnError will use the last chunk - of container log output if the termination message - file is empty and the container exited with - an error. The log output is limited to 2048 - bytes or 80 lines, whichever is smaller. Defaults - to File. Cannot be updated. - type: string - tty: - description: Whether this container should allocate - a TTY for itself, also requires 'stdin' to be - true. Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block - devices to be used by the container. - items: - description: volumeDevice describes a mapping - of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside - of the container that the device will - be mapped to. - type: string - name: - description: name must match the name of - a persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - description: Pod volumes to mount into the container's - filesystem. Cannot be updated. - items: - description: VolumeMount describes a mounting - of a Volume within a container. - properties: - mountPath: - description: Path within the container at - which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: mountPropagation determines - how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is - used. This field is beta in 1.10. - type: string - name: - description: This must match the Name of - a Volume. - type: string - readOnly: - description: Mounted read-only if true, - read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - subPath: - description: Path within the volume from - which the container's volume should be - mounted. Defaults to "" (volume's root). - type: string - subPathExpr: - description: Expanded path within the volume - from which the container's volume should - be mounted. Behaves similarly to SubPath - but environment variable references $(VAR_NAME) - are expanded using the container's environment. - Defaults to "" (volume's root). SubPathExpr - and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - description: Container's working directory. If - not specified, the container runtime's default - will be used, which might be configured in the - container image. Cannot be updated. - type: string - required: - - name - type: object - type: array - dnsConfig: - description: Specifies the DNS parameters of a pod. - Parameters specified here will be merged to the generated - DNS configuration based on DNSPolicy. - properties: - nameservers: - description: A list of DNS name server IP addresses. - This will be appended to the base nameservers - generated from DNSPolicy. Duplicated nameservers - will be removed. - items: - type: string - type: array - options: - description: A list of DNS resolver options. This - will be merged with the base options generated - from DNSPolicy. Duplicated entries will be removed. - Resolution options given in Options will override - those that appear in the base DNSPolicy. - items: - description: PodDNSConfigOption defines DNS resolver - options of a pod. - properties: - name: - description: Required. - type: string - value: - type: string - type: object - type: array - searches: - description: A list of DNS search domains for host-name - lookup. This will be appended to the base search - paths generated from DNSPolicy. Duplicated search - paths will be removed. - items: - type: string - type: array - type: object - dnsPolicy: - description: Set DNS policy for the pod. Defaults to - "ClusterFirst". Valid values are 'ClusterFirstWithHostNet', - 'ClusterFirst', 'Default' or 'None'. DNS parameters - given in DNSConfig will be merged with the policy - selected with DNSPolicy. To have DNS options set along - with hostNetwork, you have to specify DNS policy explicitly - to 'ClusterFirstWithHostNet'. - type: string - enableServiceLinks: - description: 'EnableServiceLinks indicates whether information - about services should be injected into pod''s environment - variables, matching the syntax of Docker links. Optional: - Defaults to true.' - type: boolean - ephemeralContainers: - description: List of ephemeral containers run in this - pod. Ephemeral containers may be run in an existing - pod to perform user-initiated actions such as debugging. - This list cannot be specified when creating a pod, - and it cannot be modified by updating the pod spec. - In order to add an ephemeral container to an existing - pod, use the pod's ephemeralcontainers subresource. - This field is alpha-level and is only honored by servers - that enable the EphemeralContainers feature. - items: - description: An EphemeralContainer is a container - that may be added temporarily to an existing pod - for user-initiated activities such as debugging. - Ephemeral containers have no resource or scheduling - guarantees, and they will not be restarted when - they exit or when a pod is removed or restarted. - If an ephemeral container causes a pod to exceed - its resource allocation, the pod may be evicted. - Ephemeral containers may not be added by directly - updating the pod spec. They must be added via the - pod's ephemeralcontainers subresource, and they - will appear in the pod spec once added. This is - an alpha feature enabled by the EphemeralContainers - feature flag. - properties: - args: - description: 'Arguments to the entrypoint. The - docker image''s CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded - using the container''s environment. If a variable - cannot be resolved, the reference in the input - string will be unchanged. The $(VAR_NAME) syntax - can be escaped with a double $$, ie: $$(VAR_NAME). - Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot - be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' - items: - type: string - type: array - command: - description: 'Entrypoint array. Not executed within - a shell. The docker image''s ENTRYPOINT is used - if this is not provided. Variable references - $(VAR_NAME) are expanded using the container''s - environment. If a variable cannot be resolved, - the reference in the input string will be unchanged. - The $(VAR_NAME) syntax can be escaped with a - double $$, ie: $$(VAR_NAME). Escaped references - will never be expanded, regardless of whether - the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' - items: - type: string - type: array - env: - description: List of environment variables to - set in the container. Cannot be updated. - items: - description: EnvVar represents an environment - variable present in a Container. - properties: - name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. - type: string - value: - description: 'Variable references $(VAR_NAME) - are expanded using the previous defined - environment variables in the container - and any service environment variables. - If a variable cannot be resolved, the - reference in the input string will be - unchanged. The $(VAR_NAME) syntax can - be escaped with a double $$, ie: $$(VAR_NAME). - Escaped references will never be expanded, - regardless of whether the variable exists - or not. Defaults to "".' - type: string - valueFrom: - description: Source for the environment - variable's value. Cannot be used if value - is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - description: 'Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. - apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the - ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - fieldRef: - description: 'Selects a field of the - pod: supports metadata.name, metadata.namespace, - `metadata.labels['''']`, `metadata.annotations['''']`, - spec.nodeName, spec.serviceAccountName, - status.hostIP, status.podIP, status.podIPs.' - properties: - apiVersion: - description: Version of the schema - the FieldPath is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to - select in the specified API version. - type: string - required: - - fieldPath - type: object - resourceFieldRef: - description: 'Selects a resource of - the container: only resources limits - and requests (limits.cpu, limits.memory, - limits.ephemeral-storage, requests.cpu, - requests.memory and requests.ephemeral-storage) - are currently supported.' - properties: - containerName: - description: 'Container name: required - for volumes, optional for env - vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource - to select' - type: string - required: - - resource - type: object - secretKeyRef: - description: Selects a key of a secret - in the pod's namespace - properties: - key: - description: The key of the secret - to select from. Must be a valid - secret key. - type: string - name: - description: 'Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. - apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the - Secret or its key must be defined - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - description: List of sources to populate environment - variables in the container. The keys defined - within a source must be a C_IDENTIFIER. All - invalid keys will be reported as an event when - the container is starting. When a key exists - in multiple sources, the value associated with - the last source will take precedence. Values - defined by an Env with a duplicate key will - take precedence. Cannot be updated. - items: - description: EnvFromSource represents the source - of a set of ConfigMaps - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - description: 'Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - optional: - description: Specify whether the ConfigMap - must be defined - type: boolean - type: object - prefix: - description: An optional identifier to prepend - to each key in the ConfigMap. Must be - a C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from - properties: - name: - description: 'Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - optional: - description: Specify whether the Secret - must be defined - type: boolean - type: object - type: object - type: array - image: - description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images' - type: string - imagePullPolicy: - description: 'Image pull policy. One of Always, - Never, IfNotPresent. Defaults to Always if :latest - tag is specified, or IfNotPresent otherwise. - Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' - type: string - lifecycle: - description: Lifecycle is not allowed for ephemeral - containers. - properties: - postStart: - description: 'PostStart is called immediately - after a container is created. If the handler - fails, the container is terminated and restarted - according to its restart policy. Other management - of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' - properties: - exec: - description: One and only one of the following - should be specified. Exec specifies - the action to take. - properties: - command: - description: Command is the command - line to execute inside the container, - the working directory for the command is - root ('/') in the container's filesystem. - The command is simply exec'd, it - is not run inside a shell, so traditional - shell instructions ('|', etc) won't - work. To use a shell, you need to - explicitly call out to that shell. - Exit status of 0 is treated as live/healthy - and non-zero is unhealthy. - items: - type: string - type: array - type: object - httpGet: - description: HTTPGet specifies the http - request to perform. - properties: - host: - description: Host name to connect - to, defaults to the pod IP. You - probably want to set "Host" in httpHeaders - instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: The header field - name - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the - port to access on the container. - Number must be in the range 1 to - 65535. Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - tcpSocket: - description: 'TCPSocket specifies an action - involving a TCP port. TCP hooks not - yet supported TODO: implement a realistic - TCP lifecycle hook' - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the - port to access on the container. - Number must be in the range 1 to - 65535. Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: 'PreStop is called immediately - before a container is terminated due to - an API request or management event such - as liveness/startup probe failure, preemption, - resource contention, etc. The handler is - not called if the container crashes or exits. - The reason for termination is passed to - the handler. The Pod''s termination grace - period countdown begins before the PreStop - hooked is executed. Regardless of the outcome - of the handler, the container will eventually - terminate within the Pod''s termination - grace period. Other management of the container - blocks until the hook completes or until - the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' - properties: - exec: - description: One and only one of the following - should be specified. Exec specifies - the action to take. - properties: - command: - description: Command is the command - line to execute inside the container, - the working directory for the command is - root ('/') in the container's filesystem. - The command is simply exec'd, it - is not run inside a shell, so traditional - shell instructions ('|', etc) won't - work. To use a shell, you need to - explicitly call out to that shell. - Exit status of 0 is treated as live/healthy - and non-zero is unhealthy. - items: - type: string - type: array - type: object - httpGet: - description: HTTPGet specifies the http - request to perform. - properties: - host: - description: Host name to connect - to, defaults to the pod IP. You - probably want to set "Host" in httpHeaders - instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: The header field - name - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the - port to access on the container. - Number must be in the range 1 to - 65535. Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - tcpSocket: - description: 'TCPSocket specifies an action - involving a TCP port. TCP hooks not - yet supported TODO: implement a realistic - TCP lifecycle hook' - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the - port to access on the container. - Number must be in the range 1 to - 65535. Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - description: Probes are not allowed for ephemeral - containers. - properties: - exec: - description: One and only one of the following - should be specified. Exec specifies the - action to take. - properties: - command: - description: Command is the command line - to execute inside the container, the - working directory for the command is - root ('/') in the container's filesystem. - The command is simply exec'd, it is - not run inside a shell, so traditional - shell instructions ('|', etc) won't - work. To use a shell, you need to explicitly - call out to that shell. Exit status - of 0 is treated as live/healthy and - non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: Minimum consecutive failures - for the probe to be considered failed after - having succeeded. Defaults to 3. Minimum - value is 1. - format: int32 - type: integer - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: Host name to connect to, - defaults to the pod IP. You probably - want to set "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: The header field name - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: 'Number of seconds after the - container has started before liveness probes - are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - periodSeconds: - description: How often (in seconds) to perform - the probe. Default to 10 seconds. Minimum - value is 1. - format: int32 - type: integer - successThreshold: - description: Minimum consecutive successes - for the probe to be considered successful - after having failed. Defaults to 1. Must - be 1 for liveness and startup. Minimum value - is 1. - format: int32 - type: integer - tcpSocket: - description: 'TCPSocket specifies an action - involving a TCP port. TCP hooks not yet - supported TODO: implement a realistic TCP - lifecycle hook' - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - description: 'Number of seconds after which - the probe times out. Defaults to 1 second. - Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - type: object - name: - description: Name of the ephemeral container specified - as a DNS_LABEL. This name must be unique among - all containers, init containers and ephemeral - containers. - type: string - ports: - description: Ports are not allowed for ephemeral - containers. - items: - description: ContainerPort represents a network - port in a single container. - properties: - containerPort: - description: Number of port to expose on - the pod's IP address. This must be a valid - port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external - port to. - type: string - hostPort: - description: Number of port to expose on - the host. If specified, this must be a - valid port number, 0 < x < 65536. If HostNetwork - is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: If specified, this must be - an IANA_SVC_NAME and unique within the - pod. Each named port in a pod must have - a unique name. Name for the port that - can be referred to by services. - type: string - protocol: - default: TCP - description: Protocol for port. Must be - UDP, TCP, or SCTP. Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - readinessProbe: - description: Probes are not allowed for ephemeral - containers. - properties: - exec: - description: One and only one of the following - should be specified. Exec specifies the - action to take. - properties: - command: - description: Command is the command line - to execute inside the container, the - working directory for the command is - root ('/') in the container's filesystem. - The command is simply exec'd, it is - not run inside a shell, so traditional - shell instructions ('|', etc) won't - work. To use a shell, you need to explicitly - call out to that shell. Exit status - of 0 is treated as live/healthy and - non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: Minimum consecutive failures - for the probe to be considered failed after - having succeeded. Defaults to 3. Minimum - value is 1. - format: int32 - type: integer - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: Host name to connect to, - defaults to the pod IP. You probably - want to set "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: The header field name - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: 'Number of seconds after the - container has started before liveness probes - are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - periodSeconds: - description: How often (in seconds) to perform - the probe. Default to 10 seconds. Minimum - value is 1. - format: int32 - type: integer - successThreshold: - description: Minimum consecutive successes - for the probe to be considered successful - after having failed. Defaults to 1. Must - be 1 for liveness and startup. Minimum value - is 1. - format: int32 - type: integer - tcpSocket: - description: 'TCPSocket specifies an action - involving a TCP port. TCP hooks not yet - supported TODO: implement a realistic TCP - lifecycle hook' - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - description: 'Number of seconds after which - the probe times out. Defaults to 1 second. - Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - type: object - resources: - description: Resources are not allowed for ephemeral - containers. Ephemeral containers use spare resources - already allocated to the pod. - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Limits describes the maximum - amount of compute resources allowed. More - info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes the minimum - amount of compute resources required. If - Requests is omitted for a container, it - defaults to Limits if that is explicitly - specified, otherwise to an implementation-defined - value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' - type: object - type: object - securityContext: - description: SecurityContext is not allowed for - ephemeral containers. - properties: - allowPrivilegeEscalation: - description: 'AllowPrivilegeEscalation controls - whether a process can gain more privileges - than its parent process. This bool directly - controls if the no_new_privs flag will be - set on the container process. AllowPrivilegeEscalation - is true always when the container is: 1) - run as Privileged 2) has CAP_SYS_ADMIN' - type: boolean - capabilities: - description: The capabilities to add/drop - when running containers. Defaults to the - default set of capabilities granted by the - container runtime. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - drop: - description: Removed capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - type: object - privileged: - description: Run container in privileged mode. - Processes in privileged containers are essentially - equivalent to root on the host. Defaults - to false. - type: boolean - procMount: - description: procMount denotes the type of - proc mount to use for the containers. The - default is DefaultProcMount which uses the - container runtime defaults for readonly - paths and masked paths. This requires the - ProcMountType feature flag to be enabled. - type: string - readOnlyRootFilesystem: - description: Whether this container has a - read-only root filesystem. Default is false. - type: boolean - runAsGroup: - description: The GID to run the entrypoint - of the container process. Uses runtime default - if unset. May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes - precedence. - format: int64 - type: integer - runAsNonRoot: - description: Indicates that the container - must run as a non-root user. If true, the - Kubelet will validate the image at runtime - to ensure that it does not run as UID 0 - (root) and fail to start the container if - it does. If unset or false, no such validation - will be performed. May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes - precedence. - type: boolean - runAsUser: - description: The UID to run the entrypoint - of the container process. Defaults to user - specified in image metadata if unspecified. - May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes - precedence. - format: int64 - type: integer - seLinuxOptions: - description: The SELinux context to be applied - to the container. If unspecified, the container - runtime will allocate a random SELinux context - for each container. May also be set in - PodSecurityContext. If set in both SecurityContext - and PodSecurityContext, the value specified - in SecurityContext takes precedence. - properties: - level: - description: Level is SELinux level label - that applies to the container. - type: string - role: - description: Role is a SELinux role label - that applies to the container. - type: string - type: - description: Type is a SELinux type label - that applies to the container. - type: string - user: - description: User is a SELinux user label - that applies to the container. - type: string - type: object - seccompProfile: - description: The seccomp options to use by - this container. If seccomp options are provided - at both the pod & container level, the container - options override the pod options. - properties: - localhostProfile: - description: localhostProfile indicates - a profile defined in a file on the node - should be used. The profile must be - preconfigured on the node to work. Must - be a descending path, relative to the - kubelet's configured seccomp profile - location. Must only be set if type is - "Localhost". - type: string - type: - description: "type indicates which kind - of seccomp profile will be applied. - Valid options are: \n Localhost - a - profile defined in a file on the node - should be used. RuntimeDefault - the - container runtime default profile should - be used. Unconfined - no profile should - be applied." - type: string - required: - - type - type: object - windowsOptions: - description: The Windows specific settings - applied to all containers. If unspecified, - the options from the PodSecurityContext - will be used. If set in both SecurityContext - and PodSecurityContext, the value specified - in SecurityContext takes precedence. - properties: - gmsaCredentialSpec: - description: GMSACredentialSpec is where - the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) - inlines the contents of the GMSA credential - spec named by the GMSACredentialSpecName - field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is - the name of the GMSA credential spec - to use. - type: string - runAsUserName: - description: The UserName in Windows to - run the entrypoint of the container - process. Defaults to the user specified - in image metadata if unspecified. May - also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext - takes precedence. - type: string - type: object - type: object - startupProbe: - description: Probes are not allowed for ephemeral - containers. - properties: - exec: - description: One and only one of the following - should be specified. Exec specifies the - action to take. - properties: - command: - description: Command is the command line - to execute inside the container, the - working directory for the command is - root ('/') in the container's filesystem. - The command is simply exec'd, it is - not run inside a shell, so traditional - shell instructions ('|', etc) won't - work. To use a shell, you need to explicitly - call out to that shell. Exit status - of 0 is treated as live/healthy and - non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: Minimum consecutive failures - for the probe to be considered failed after - having succeeded. Defaults to 3. Minimum - value is 1. - format: int32 - type: integer - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: Host name to connect to, - defaults to the pod IP. You probably - want to set "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: The header field name - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: 'Number of seconds after the - container has started before liveness probes - are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - periodSeconds: - description: How often (in seconds) to perform - the probe. Default to 10 seconds. Minimum - value is 1. - format: int32 - type: integer - successThreshold: - description: Minimum consecutive successes - for the probe to be considered successful - after having failed. Defaults to 1. Must - be 1 for liveness and startup. Minimum value - is 1. - format: int32 - type: integer - tcpSocket: - description: 'TCPSocket specifies an action - involving a TCP port. TCP hooks not yet - supported TODO: implement a realistic TCP - lifecycle hook' - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - description: 'Number of seconds after which - the probe times out. Defaults to 1 second. - Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - type: object - stdin: - description: Whether this container should allocate - a buffer for stdin in the container runtime. - If this is not set, reads from stdin in the - container will always result in EOF. Default - is false. - type: boolean - stdinOnce: - description: Whether the container runtime should - close the stdin channel after it has been opened - by a single attach. When stdin is true the stdin - stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin - is opened on container start, is empty until - the first client attaches to stdin, and then - remains open and accepts data until the client - disconnects, at which time stdin is closed and - remains closed until the container is restarted. - If this flag is false, a container processes - that reads from stdin will never receive an - EOF. Default is false - type: boolean - targetContainerName: - description: If set, the name of the container - from PodSpec that this ephemeral container targets. - The ephemeral container will be run in the namespaces - (IPC, PID, etc) of this container. If not set - then the ephemeral container is run in whatever - namespaces are shared for the pod. Note that - the container runtime must support this feature. - type: string - terminationMessagePath: - description: 'Optional: Path at which the file - to which the container''s termination message - will be written is mounted into the container''s - filesystem. Message written is intended to be - brief final status, such as an assertion failure - message. Will be truncated by the node if greater - than 4096 bytes. The total message length across - all containers will be limited to 12kb. Defaults - to /dev/termination-log. Cannot be updated.' - type: string - terminationMessagePolicy: - description: Indicate how the termination message - should be populated. File will use the contents - of terminationMessagePath to populate the container - status message on both success and failure. - FallbackToLogsOnError will use the last chunk - of container log output if the termination message - file is empty and the container exited with - an error. The log output is limited to 2048 - bytes or 80 lines, whichever is smaller. Defaults - to File. Cannot be updated. - type: string - tty: - description: Whether this container should allocate - a TTY for itself, also requires 'stdin' to be - true. Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block - devices to be used by the container. - items: - description: volumeDevice describes a mapping - of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside - of the container that the device will - be mapped to. - type: string - name: - description: name must match the name of - a persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - description: Pod volumes to mount into the container's - filesystem. Cannot be updated. - items: - description: VolumeMount describes a mounting - of a Volume within a container. - properties: - mountPath: - description: Path within the container at - which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: mountPropagation determines - how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is - used. This field is beta in 1.10. - type: string - name: - description: This must match the Name of - a Volume. - type: string - readOnly: - description: Mounted read-only if true, - read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - subPath: - description: Path within the volume from - which the container's volume should be - mounted. Defaults to "" (volume's root). - type: string - subPathExpr: - description: Expanded path within the volume - from which the container's volume should - be mounted. Behaves similarly to SubPath - but environment variable references $(VAR_NAME) - are expanded using the container's environment. - Defaults to "" (volume's root). SubPathExpr - and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - description: Container's working directory. If - not specified, the container runtime's default - will be used, which might be configured in the - container image. Cannot be updated. - type: string - required: - - name - type: object - type: array - hostAliases: - description: HostAliases is an optional list of hosts - and IPs that will be injected into the pod's hosts - file if specified. This is only valid for non-hostNetwork - pods. - items: - description: HostAlias holds the mapping between IP - and hostnames that will be injected as an entry - in the pod's hosts file. - properties: - hostnames: - description: Hostnames for the above IP address. - items: - type: string - type: array - ip: - description: IP address of the host file entry. - type: string - type: object - type: array - hostIPC: - description: 'Use the host''s ipc namespace. Optional: - Default to false.' - type: boolean - hostNetwork: - description: Host networking requested for this pod. - Use the host's network namespace. If this option is - set, the ports that will be used must be specified. - Default to false. - type: boolean - hostPID: - description: 'Use the host''s pid namespace. Optional: - Default to false.' - type: boolean - hostname: - description: Specifies the hostname of the Pod If not - specified, the pod's hostname will be set to a system-defined - value. - type: string - imagePullSecrets: - description: 'ImagePullSecrets is an optional list of - references to secrets in the same namespace to use - for pulling any of the images used by this PodSpec. - If specified, these secrets will be passed to individual - puller implementations for them to use. For example, - in the case of docker, only DockerConfig type secrets - are honored. More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod' - items: - description: LocalObjectReference contains enough - information to let you locate the referenced object - inside the same namespace. - properties: - name: - description: 'Name of the referent. More info: - https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - type: array - initContainers: - description: 'List of initialization containers belonging - to the pod. Init containers are executed in order - prior to containers being started. If any init container - fails, the pod is considered to have failed and is - handled according to its restartPolicy. The name for - an init container or normal container must be unique - among all containers. Init containers may not have - Lifecycle actions, Readiness probes, Liveness probes, - or Startup probes. The resourceRequirements of an - init container are taken into account during scheduling - by finding the highest request/limit for each resource - type, and then using the max of of that value or the - sum of the normal containers. Limits are applied to - init containers in a similar fashion. Init containers - cannot currently be added or removed. Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/' - items: - description: A single application container that you - want to run within a pod. - properties: - args: - description: 'Arguments to the entrypoint. The - docker image''s CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded - using the container''s environment. If a variable - cannot be resolved, the reference in the input - string will be unchanged. The $(VAR_NAME) syntax - can be escaped with a double $$, ie: $$(VAR_NAME). - Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot - be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' - items: - type: string - type: array - command: - description: 'Entrypoint array. Not executed within - a shell. The docker image''s ENTRYPOINT is used - if this is not provided. Variable references - $(VAR_NAME) are expanded using the container''s - environment. If a variable cannot be resolved, - the reference in the input string will be unchanged. - The $(VAR_NAME) syntax can be escaped with a - double $$, ie: $$(VAR_NAME). Escaped references - will never be expanded, regardless of whether - the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' - items: - type: string - type: array - env: - description: List of environment variables to - set in the container. Cannot be updated. - items: - description: EnvVar represents an environment - variable present in a Container. - properties: - name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. - type: string - value: - description: 'Variable references $(VAR_NAME) - are expanded using the previous defined - environment variables in the container - and any service environment variables. - If a variable cannot be resolved, the - reference in the input string will be - unchanged. The $(VAR_NAME) syntax can - be escaped with a double $$, ie: $$(VAR_NAME). - Escaped references will never be expanded, - regardless of whether the variable exists - or not. Defaults to "".' - type: string - valueFrom: - description: Source for the environment - variable's value. Cannot be used if value - is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - description: 'Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. - apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the - ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - fieldRef: - description: 'Selects a field of the - pod: supports metadata.name, metadata.namespace, - `metadata.labels['''']`, `metadata.annotations['''']`, - spec.nodeName, spec.serviceAccountName, - status.hostIP, status.podIP, status.podIPs.' - properties: - apiVersion: - description: Version of the schema - the FieldPath is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to - select in the specified API version. - type: string - required: - - fieldPath - type: object - resourceFieldRef: - description: 'Selects a resource of - the container: only resources limits - and requests (limits.cpu, limits.memory, - limits.ephemeral-storage, requests.cpu, - requests.memory and requests.ephemeral-storage) - are currently supported.' - properties: - containerName: - description: 'Container name: required - for volumes, optional for env - vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource - to select' - type: string - required: - - resource - type: object - secretKeyRef: - description: Selects a key of a secret - in the pod's namespace - properties: - key: - description: The key of the secret - to select from. Must be a valid - secret key. - type: string - name: - description: 'Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. - apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the - Secret or its key must be defined - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - description: List of sources to populate environment - variables in the container. The keys defined - within a source must be a C_IDENTIFIER. All - invalid keys will be reported as an event when - the container is starting. When a key exists - in multiple sources, the value associated with - the last source will take precedence. Values - defined by an Env with a duplicate key will - take precedence. Cannot be updated. - items: - description: EnvFromSource represents the source - of a set of ConfigMaps - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - description: 'Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - optional: - description: Specify whether the ConfigMap - must be defined - type: boolean - type: object - prefix: - description: An optional identifier to prepend - to each key in the ConfigMap. Must be - a C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from - properties: - name: - description: 'Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - optional: - description: Specify whether the Secret - must be defined - type: boolean - type: object - type: object - type: array - image: - description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level - config management to default or override container - images in workload controllers like Deployments - and StatefulSets.' - type: string - imagePullPolicy: - description: 'Image pull policy. One of Always, - Never, IfNotPresent. Defaults to Always if :latest - tag is specified, or IfNotPresent otherwise. - Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' - type: string - lifecycle: - description: Actions that the management system - should take in response to container lifecycle - events. Cannot be updated. - properties: - postStart: - description: 'PostStart is called immediately - after a container is created. If the handler - fails, the container is terminated and restarted - according to its restart policy. Other management - of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' - properties: - exec: - description: One and only one of the following - should be specified. Exec specifies - the action to take. - properties: - command: - description: Command is the command - line to execute inside the container, - the working directory for the command is - root ('/') in the container's filesystem. - The command is simply exec'd, it - is not run inside a shell, so traditional - shell instructions ('|', etc) won't - work. To use a shell, you need to - explicitly call out to that shell. - Exit status of 0 is treated as live/healthy - and non-zero is unhealthy. - items: - type: string - type: array - type: object - httpGet: - description: HTTPGet specifies the http - request to perform. - properties: - host: - description: Host name to connect - to, defaults to the pod IP. You - probably want to set "Host" in httpHeaders - instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: The header field - name - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the - port to access on the container. - Number must be in the range 1 to - 65535. Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - tcpSocket: - description: 'TCPSocket specifies an action - involving a TCP port. TCP hooks not - yet supported TODO: implement a realistic - TCP lifecycle hook' - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the - port to access on the container. - Number must be in the range 1 to - 65535. Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: 'PreStop is called immediately - before a container is terminated due to - an API request or management event such - as liveness/startup probe failure, preemption, - resource contention, etc. The handler is - not called if the container crashes or exits. - The reason for termination is passed to - the handler. The Pod''s termination grace - period countdown begins before the PreStop - hooked is executed. Regardless of the outcome - of the handler, the container will eventually - terminate within the Pod''s termination - grace period. Other management of the container - blocks until the hook completes or until - the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' - properties: - exec: - description: One and only one of the following - should be specified. Exec specifies - the action to take. - properties: - command: - description: Command is the command - line to execute inside the container, - the working directory for the command is - root ('/') in the container's filesystem. - The command is simply exec'd, it - is not run inside a shell, so traditional - shell instructions ('|', etc) won't - work. To use a shell, you need to - explicitly call out to that shell. - Exit status of 0 is treated as live/healthy - and non-zero is unhealthy. - items: - type: string - type: array - type: object - httpGet: - description: HTTPGet specifies the http - request to perform. - properties: - host: - description: Host name to connect - to, defaults to the pod IP. You - probably want to set "Host" in httpHeaders - instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: The header field - name - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the - port to access on the container. - Number must be in the range 1 to - 65535. Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - tcpSocket: - description: 'TCPSocket specifies an action - involving a TCP port. TCP hooks not - yet supported TODO: implement a realistic - TCP lifecycle hook' - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the - port to access on the container. - Number must be in the range 1 to - 65535. Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - description: 'Periodic probe of container liveness. - Container will be restarted if the probe fails. - Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - properties: - exec: - description: One and only one of the following - should be specified. Exec specifies the - action to take. - properties: - command: - description: Command is the command line - to execute inside the container, the - working directory for the command is - root ('/') in the container's filesystem. - The command is simply exec'd, it is - not run inside a shell, so traditional - shell instructions ('|', etc) won't - work. To use a shell, you need to explicitly - call out to that shell. Exit status - of 0 is treated as live/healthy and - non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: Minimum consecutive failures - for the probe to be considered failed after - having succeeded. Defaults to 3. Minimum - value is 1. - format: int32 - type: integer - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: Host name to connect to, - defaults to the pod IP. You probably - want to set "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: The header field name - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: 'Number of seconds after the - container has started before liveness probes - are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - periodSeconds: - description: How often (in seconds) to perform - the probe. Default to 10 seconds. Minimum - value is 1. - format: int32 - type: integer - successThreshold: - description: Minimum consecutive successes - for the probe to be considered successful - after having failed. Defaults to 1. Must - be 1 for liveness and startup. Minimum value - is 1. - format: int32 - type: integer - tcpSocket: - description: 'TCPSocket specifies an action - involving a TCP port. TCP hooks not yet - supported TODO: implement a realistic TCP - lifecycle hook' - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - description: 'Number of seconds after which - the probe times out. Defaults to 1 second. - Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - type: object - name: - description: Name of the container specified as - a DNS_LABEL. Each container in a pod must have - a unique name (DNS_LABEL). Cannot be updated. - type: string - ports: - description: List of ports to expose from the - container. Exposing a port here gives the system - additional information about the network connections - a container uses, but is primarily informational. - Not specifying a port here DOES NOT prevent - that port from being exposed. Any port which - is listening on the default "0.0.0.0" address - inside a container will be accessible from the - network. Cannot be updated. - items: - description: ContainerPort represents a network - port in a single container. - properties: - containerPort: - description: Number of port to expose on - the pod's IP address. This must be a valid - port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external - port to. - type: string - hostPort: - description: Number of port to expose on - the host. If specified, this must be a - valid port number, 0 < x < 65536. If HostNetwork - is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: If specified, this must be - an IANA_SVC_NAME and unique within the - pod. Each named port in a pod must have - a unique name. Name for the port that - can be referred to by services. - type: string - protocol: - default: TCP - description: Protocol for port. Must be - UDP, TCP, or SCTP. Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: 'Periodic probe of container service - readiness. Container will be removed from service - endpoints if the probe fails. Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - properties: - exec: - description: One and only one of the following - should be specified. Exec specifies the - action to take. - properties: - command: - description: Command is the command line - to execute inside the container, the - working directory for the command is - root ('/') in the container's filesystem. - The command is simply exec'd, it is - not run inside a shell, so traditional - shell instructions ('|', etc) won't - work. To use a shell, you need to explicitly - call out to that shell. Exit status - of 0 is treated as live/healthy and - non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: Minimum consecutive failures - for the probe to be considered failed after - having succeeded. Defaults to 3. Minimum - value is 1. - format: int32 - type: integer - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: Host name to connect to, - defaults to the pod IP. You probably - want to set "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: The header field name - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: 'Number of seconds after the - container has started before liveness probes - are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - periodSeconds: - description: How often (in seconds) to perform - the probe. Default to 10 seconds. Minimum - value is 1. - format: int32 - type: integer - successThreshold: - description: Minimum consecutive successes - for the probe to be considered successful - after having failed. Defaults to 1. Must - be 1 for liveness and startup. Minimum value - is 1. - format: int32 - type: integer - tcpSocket: - description: 'TCPSocket specifies an action - involving a TCP port. TCP hooks not yet - supported TODO: implement a realistic TCP - lifecycle hook' - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - description: 'Number of seconds after which - the probe times out. Defaults to 1 second. - Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - type: object - resources: - description: 'Compute Resources required by this - container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Limits describes the maximum - amount of compute resources allowed. More - info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes the minimum - amount of compute resources required. If - Requests is omitted for a container, it - defaults to Limits if that is explicitly - specified, otherwise to an implementation-defined - value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' - type: object - type: object - securityContext: - description: 'Security options the pod should - run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ - More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' - properties: - allowPrivilegeEscalation: - description: 'AllowPrivilegeEscalation controls - whether a process can gain more privileges - than its parent process. This bool directly - controls if the no_new_privs flag will be - set on the container process. AllowPrivilegeEscalation - is true always when the container is: 1) - run as Privileged 2) has CAP_SYS_ADMIN' - type: boolean - capabilities: - description: The capabilities to add/drop - when running containers. Defaults to the - default set of capabilities granted by the - container runtime. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - drop: - description: Removed capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - type: object - privileged: - description: Run container in privileged mode. - Processes in privileged containers are essentially - equivalent to root on the host. Defaults - to false. - type: boolean - procMount: - description: procMount denotes the type of - proc mount to use for the containers. The - default is DefaultProcMount which uses the - container runtime defaults for readonly - paths and masked paths. This requires the - ProcMountType feature flag to be enabled. - type: string - readOnlyRootFilesystem: - description: Whether this container has a - read-only root filesystem. Default is false. - type: boolean - runAsGroup: - description: The GID to run the entrypoint - of the container process. Uses runtime default - if unset. May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes - precedence. - format: int64 - type: integer - runAsNonRoot: - description: Indicates that the container - must run as a non-root user. If true, the - Kubelet will validate the image at runtime - to ensure that it does not run as UID 0 - (root) and fail to start the container if - it does. If unset or false, no such validation - will be performed. May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes - precedence. - type: boolean - runAsUser: - description: The UID to run the entrypoint - of the container process. Defaults to user - specified in image metadata if unspecified. - May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes - precedence. - format: int64 - type: integer - seLinuxOptions: - description: The SELinux context to be applied - to the container. If unspecified, the container - runtime will allocate a random SELinux context - for each container. May also be set in - PodSecurityContext. If set in both SecurityContext - and PodSecurityContext, the value specified - in SecurityContext takes precedence. - properties: - level: - description: Level is SELinux level label - that applies to the container. - type: string - role: - description: Role is a SELinux role label - that applies to the container. - type: string - type: - description: Type is a SELinux type label - that applies to the container. - type: string - user: - description: User is a SELinux user label - that applies to the container. - type: string - type: object - seccompProfile: - description: The seccomp options to use by - this container. If seccomp options are provided - at both the pod & container level, the container - options override the pod options. - properties: - localhostProfile: - description: localhostProfile indicates - a profile defined in a file on the node - should be used. The profile must be - preconfigured on the node to work. Must - be a descending path, relative to the - kubelet's configured seccomp profile - location. Must only be set if type is - "Localhost". - type: string - type: - description: "type indicates which kind - of seccomp profile will be applied. - Valid options are: \n Localhost - a - profile defined in a file on the node - should be used. RuntimeDefault - the - container runtime default profile should - be used. Unconfined - no profile should - be applied." - type: string - required: - - type - type: object - windowsOptions: - description: The Windows specific settings - applied to all containers. If unspecified, - the options from the PodSecurityContext - will be used. If set in both SecurityContext - and PodSecurityContext, the value specified - in SecurityContext takes precedence. - properties: - gmsaCredentialSpec: - description: GMSACredentialSpec is where - the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) - inlines the contents of the GMSA credential - spec named by the GMSACredentialSpecName - field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is - the name of the GMSA credential spec - to use. - type: string - runAsUserName: - description: The UserName in Windows to - run the entrypoint of the container - process. Defaults to the user specified - in image metadata if unspecified. May - also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext - takes precedence. - type: string - type: object - type: object - startupProbe: - description: 'StartupProbe indicates that the - Pod has successfully initialized. If specified, - no other probes are executed until this completes - successfully. If this probe fails, the Pod will - be restarted, just as if the livenessProbe failed. - This can be used to provide different probe - parameters at the beginning of a Pod''s lifecycle, - when it might take a long time to load data - or warm a cache, than during steady-state operation. - This cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - properties: - exec: - description: One and only one of the following - should be specified. Exec specifies the - action to take. - properties: - command: - description: Command is the command line - to execute inside the container, the - working directory for the command is - root ('/') in the container's filesystem. - The command is simply exec'd, it is - not run inside a shell, so traditional - shell instructions ('|', etc) won't - work. To use a shell, you need to explicitly - call out to that shell. Exit status - of 0 is treated as live/healthy and - non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: Minimum consecutive failures - for the probe to be considered failed after - having succeeded. Defaults to 3. Minimum - value is 1. - format: int32 - type: integer - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: Host name to connect to, - defaults to the pod IP. You probably - want to set "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: The header field name - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: 'Number of seconds after the - container has started before liveness probes - are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - periodSeconds: - description: How often (in seconds) to perform - the probe. Default to 10 seconds. Minimum - value is 1. - format: int32 - type: integer - successThreshold: - description: Minimum consecutive successes - for the probe to be considered successful - after having failed. Defaults to 1. Must - be 1 for liveness and startup. Minimum value - is 1. - format: int32 - type: integer - tcpSocket: - description: 'TCPSocket specifies an action - involving a TCP port. TCP hooks not yet - supported TODO: implement a realistic TCP - lifecycle hook' - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the port - to access on the container. Number must - be in the range 1 to 65535. Name must - be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - description: 'Number of seconds after which - the probe times out. Defaults to 1 second. - Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - type: object - stdin: - description: Whether this container should allocate - a buffer for stdin in the container runtime. - If this is not set, reads from stdin in the - container will always result in EOF. Default - is false. - type: boolean - stdinOnce: - description: Whether the container runtime should - close the stdin channel after it has been opened - by a single attach. When stdin is true the stdin - stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin - is opened on container start, is empty until - the first client attaches to stdin, and then - remains open and accepts data until the client - disconnects, at which time stdin is closed and - remains closed until the container is restarted. - If this flag is false, a container processes - that reads from stdin will never receive an - EOF. Default is false - type: boolean - terminationMessagePath: - description: 'Optional: Path at which the file - to which the container''s termination message - will be written is mounted into the container''s - filesystem. Message written is intended to be - brief final status, such as an assertion failure - message. Will be truncated by the node if greater - than 4096 bytes. The total message length across - all containers will be limited to 12kb. Defaults - to /dev/termination-log. Cannot be updated.' - type: string - terminationMessagePolicy: - description: Indicate how the termination message - should be populated. File will use the contents - of terminationMessagePath to populate the container - status message on both success and failure. - FallbackToLogsOnError will use the last chunk - of container log output if the termination message - file is empty and the container exited with - an error. The log output is limited to 2048 - bytes or 80 lines, whichever is smaller. Defaults - to File. Cannot be updated. - type: string - tty: - description: Whether this container should allocate - a TTY for itself, also requires 'stdin' to be - true. Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block - devices to be used by the container. - items: - description: volumeDevice describes a mapping - of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside - of the container that the device will - be mapped to. - type: string - name: - description: name must match the name of - a persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - description: Pod volumes to mount into the container's - filesystem. Cannot be updated. - items: - description: VolumeMount describes a mounting - of a Volume within a container. - properties: - mountPath: - description: Path within the container at - which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: mountPropagation determines - how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is - used. This field is beta in 1.10. - type: string - name: - description: This must match the Name of - a Volume. - type: string - readOnly: - description: Mounted read-only if true, - read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - subPath: - description: Path within the volume from - which the container's volume should be - mounted. Defaults to "" (volume's root). - type: string - subPathExpr: - description: Expanded path within the volume - from which the container's volume should - be mounted. Behaves similarly to SubPath - but environment variable references $(VAR_NAME) - are expanded using the container's environment. - Defaults to "" (volume's root). SubPathExpr - and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - description: Container's working directory. If - not specified, the container runtime's default - will be used, which might be configured in the - container image. Cannot be updated. - type: string - required: - - name - type: object - type: array - nodeName: - description: NodeName is a request to schedule this - pod onto a specific node. If it is non-empty, the - scheduler simply schedules this pod onto that node, - assuming that it fits resource requirements. - type: string - nodeSelector: - additionalProperties: - type: string - description: 'NodeSelector is a selector which must - be true for the pod to fit on a node. Selector which - must match a node''s labels for the pod to be scheduled - on that node. More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/' - type: object - overhead: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Overhead represents the resource overhead - associated with running a pod for a given RuntimeClass. - This field will be autopopulated at admission time - by the RuntimeClass admission controller. If the RuntimeClass - admission controller is enabled, overhead must not - be set in Pod create requests. The RuntimeClass admission - controller will reject Pod create requests which have - the overhead already set. If RuntimeClass is configured - and selected in the PodSpec, Overhead will be set - to the value defined in the corresponding RuntimeClass, - otherwise it will remain unset and treated as zero. - More info: https://git.k8s.io/enhancements/keps/sig-node/20190226-pod-overhead.md - This field is alpha-level as of Kubernetes v1.16, - and is only honored by servers that enable the PodOverhead - feature.' - type: object - preemptionPolicy: - description: PreemptionPolicy is the Policy for preempting - pods with lower priority. One of Never, PreemptLowerPriority. - Defaults to PreemptLowerPriority if unset. This field - is beta-level, gated by the NonPreemptingPriority - feature-gate. - type: string - priority: - description: The priority value. Various system components - use this field to find the priority of the pod. When - Priority Admission Controller is enabled, it prevents - users from setting this field. The admission controller - populates this field from PriorityClassName. The higher - the value, the higher the priority. - format: int32 - type: integer - priorityClassName: - description: If specified, indicates the pod's priority. - "system-node-critical" and "system-cluster-critical" - are two special keywords which indicate the highest - priorities with the former being the highest priority. - Any other name must be defined by creating a PriorityClass - object with that name. If not specified, the pod priority - will be default or zero if there is no default. - type: string - readinessGates: - description: 'If specified, all readiness gates will - be evaluated for pod readiness. A pod is ready when - all its containers are ready AND all conditions specified - in the readiness gates have status equal to "True" - More info: https://git.k8s.io/enhancements/keps/sig-network/0007-pod-ready%2B%2B.md' - items: - description: PodReadinessGate contains the reference - to a pod condition - properties: - conditionType: - description: ConditionType refers to a condition - in the pod's condition list with matching type. - type: string - required: - - conditionType - type: object - type: array - restartPolicy: - description: 'Restart policy for all containers within - the pod. One of Always, OnFailure, Never. Default - to Always. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy' - type: string - runtimeClassName: - description: 'RuntimeClassName refers to a RuntimeClass - object in the node.k8s.io group, which should be used - to run this pod. If no RuntimeClass resource matches - the named class, the pod will not be run. If unset - or empty, the "legacy" RuntimeClass will be used, - which is an implicit class with an empty definition - that uses the default runtime handler. More info: - https://git.k8s.io/enhancements/keps/sig-node/runtime-class.md - This is a beta feature as of Kubernetes v1.14.' - type: string - schedulerName: - description: If specified, the pod will be dispatched - by specified scheduler. If not specified, the pod - will be dispatched by default scheduler. - type: string - securityContext: - description: 'SecurityContext holds pod-level security - attributes and common container settings. Optional: - Defaults to empty. See type description for default - values of each field.' - properties: - fsGroup: - description: "A special supplemental group that - applies to all containers in a pod. Some volume - types allow the Kubelet to change the ownership - of that volume to be owned by the pod: \n 1. The - owning GID will be the FSGroup 2. The setgid bit - is set (new files created in the volume will be - owned by FSGroup) 3. The permission bits are OR'd - with rw-rw---- \n If unset, the Kubelet will not - modify the ownership and permissions of any volume." - format: int64 - type: integer - fsGroupChangePolicy: - description: 'fsGroupChangePolicy defines behavior - of changing ownership and permission of the volume - before being exposed inside Pod. This field will - only apply to volume types which support fsGroup - based ownership(and permissions). It will have - no effect on ephemeral volume types such as: secret, - configmaps and emptydir. Valid values are "OnRootMismatch" - and "Always". If not specified, "Always" is used.' - type: string - runAsGroup: - description: The GID to run the entrypoint of the - container process. Uses runtime default if unset. - May also be set in SecurityContext. If set in - both SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence - for that container. - format: int64 - type: integer - runAsNonRoot: - description: Indicates that the container must run - as a non-root user. If true, the Kubelet will - validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start - the container if it does. If unset or false, no - such validation will be performed. May also be - set in SecurityContext. If set in both SecurityContext - and PodSecurityContext, the value specified in - SecurityContext takes precedence. - type: boolean - runAsUser: - description: The UID to run the entrypoint of the - container process. Defaults to user specified - in image metadata if unspecified. May also be - set in SecurityContext. If set in both SecurityContext - and PodSecurityContext, the value specified in - SecurityContext takes precedence for that container. - format: int64 - type: integer - seLinuxOptions: - description: The SELinux context to be applied to - all containers. If unspecified, the container - runtime will allocate a random SELinux context - for each container. May also be set in SecurityContext. If - set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes precedence - for that container. - properties: - level: - description: Level is SELinux level label that - applies to the container. - type: string - role: - description: Role is a SELinux role label that - applies to the container. - type: string - type: - description: Type is a SELinux type label that - applies to the container. - type: string - user: - description: User is a SELinux user label that - applies to the container. - type: string - type: object - seccompProfile: - description: The seccomp options to use by the containers - in this pod. - properties: - localhostProfile: - description: localhostProfile indicates a profile - defined in a file on the node should be used. - The profile must be preconfigured on the node - to work. Must be a descending path, relative - to the kubelet's configured seccomp profile - location. Must only be set if type is "Localhost". - type: string - type: - description: "type indicates which kind of seccomp - profile will be applied. Valid options are: - \n Localhost - a profile defined in a file - on the node should be used. RuntimeDefault - - the container runtime default profile should - be used. Unconfined - no profile should be - applied." - type: string - required: - - type - type: object - supplementalGroups: - description: A list of groups applied to the first - process run in each container, in addition to - the container's primary GID. If unspecified, - no groups will be added to any container. - items: - format: int64 - type: integer - type: array - sysctls: - description: Sysctls hold a list of namespaced sysctls - used for the pod. Pods with unsupported sysctls - (by the container runtime) might fail to launch. - items: - description: Sysctl defines a kernel parameter - to be set - properties: - name: - description: Name of a property to set - type: string - value: - description: Value of a property to set - type: string - required: - - name - - value - type: object - type: array - windowsOptions: - description: The Windows specific settings applied - to all containers. If unspecified, the options - within a container's SecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes precedence. - properties: - gmsaCredentialSpec: - description: GMSACredentialSpec is where the - GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) - inlines the contents of the GMSA credential - spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name - of the GMSA credential spec to use. - type: string - runAsUserName: - description: The UserName in Windows to run - the entrypoint of the container process. Defaults - to the user specified in image metadata if - unspecified. May also be set in PodSecurityContext. - If set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes - precedence. - type: string - type: object - type: object - serviceAccount: - description: 'DeprecatedServiceAccount is a depreciated - alias for ServiceAccountName. Deprecated: Use serviceAccountName - instead.' - type: string - serviceAccountName: - description: 'ServiceAccountName is the name of the - ServiceAccount to use to run this pod. More info: - https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/' - type: string - setHostnameAsFQDN: - description: If true the pod's hostname will be configured - as the pod's FQDN, rather than the leaf name (the - default). In Linux containers, this means setting - the FQDN in the hostname field of the kernel (the - nodename field of struct utsname). In Windows containers, - this means setting the registry value of hostname - for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters - to FQDN. If a pod does not have FQDN, this has no - effect. Default to false. - type: boolean - shareProcessNamespace: - description: 'Share a single process namespace between - all of the containers in a pod. When this is set containers - will be able to view and signal processes from other - containers in the same pod, and the first process - in each container will not be assigned PID 1. HostPID - and ShareProcessNamespace cannot both be set. Optional: - Default to false.' - type: boolean - subdomain: - description: If specified, the fully qualified Pod hostname - will be "...svc.". If not specified, the pod will not have - a domainname at all. - type: string - terminationGracePeriodSeconds: - description: Optional duration in seconds the pod needs - to terminate gracefully. May be decreased in delete - request. Value must be non-negative integer. The value - zero indicates delete immediately. If this value is - nil, the default grace period will be used instead. - The grace period is the duration in seconds after - the processes running in the pod are sent a termination - signal and the time when the processes are forcibly - halted with a kill signal. Set this value longer than - the expected cleanup time for your process. Defaults - to 30 seconds. - format: int64 - type: integer - tolerations: - description: If specified, the pod's tolerations. - items: - description: The pod this Toleration is attached to - tolerates any taint that matches the triple - using the matching operator . - properties: - effect: - description: Effect indicates the taint effect - to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, - PreferNoSchedule and NoExecute. - type: string - key: - description: Key is the taint key that the toleration - applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; - this combination means to match all values and - all keys. - type: string - operator: - description: Operator represents a key's relationship - to the value. Valid operators are Exists and - Equal. Defaults to Equal. Exists is equivalent - to wildcard for value, so that a pod can tolerate - all taints of a particular category. - type: string - tolerationSeconds: - description: TolerationSeconds represents the - period of time the toleration (which must be - of effect NoExecute, otherwise this field is - ignored) tolerates the taint. By default, it - is not set, which means tolerate the taint forever - (do not evict). Zero and negative values will - be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: Value is the taint value the toleration - matches to. If the operator is Exists, the value - should be empty, otherwise just a regular string. - type: string - type: object - type: array - topologySpreadConstraints: - description: TopologySpreadConstraints describes how - a group of pods ought to spread across topology domains. - Scheduler will schedule pods in a way which abides - by the constraints. All topologySpreadConstraints - are ANDed. - items: - description: TopologySpreadConstraint specifies how - to spread matching pods among the given topology. - properties: - labelSelector: - description: LabelSelector is used to find matching - pods. Pods that match this label selector are - counted to determine the number of pods in their - corresponding topology domain. - properties: - matchExpressions: - description: matchExpressions is a list of - label selector requirements. The requirements - are ANDed. - items: - description: A label selector requirement - is a selector that contains values, a - key, and an operator that relates the - key and values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only - "value". The requirements are ANDed. - type: object - type: object - maxSkew: - description: 'MaxSkew describes the degree to - which pods may be unevenly distributed. When - `whenUnsatisfiable=DoNotSchedule`, it is the - maximum permitted difference between the number - of matching pods in the target topology and - the global minimum. For example, in a 3-zone - cluster, MaxSkew is set to 1, and pods with - the same labelSelector spread as 1/1/0: | zone1 - | zone2 | zone3 | | P | P | | - - if MaxSkew is 1, incoming pod can only be - scheduled to zone3 to become 1/1/1; scheduling - it onto zone1(zone2) would make the ActualSkew(2-0) - on zone1(zone2) violate MaxSkew(1). - if MaxSkew - is 2, incoming pod can be scheduled onto any - zone. When `whenUnsatisfiable=ScheduleAnyway`, - it is used to give higher precedence to topologies - that satisfy it. It''s a required field. Default - value is 1 and 0 is not allowed.' - format: int32 - type: integer - topologyKey: - description: TopologyKey is the key of node labels. - Nodes that have a label with this key and identical - values are considered to be in the same topology. - We consider each as a "bucket", - and try to put balanced number of pods into - each bucket. It's a required field. - type: string - whenUnsatisfiable: - description: 'WhenUnsatisfiable indicates how - to deal with a pod if it doesn''t satisfy the - spread constraint. - DoNotSchedule (default) - tells the scheduler not to schedule it. - ScheduleAnyway - tells the scheduler to schedule the pod in any - location, but giving higher precedence to - topologies that would help reduce the skew. - A constraint is considered "Unsatisfiable" for - an incoming pod if and only if every possible - node assigment for that pod would violate "MaxSkew" - on some topology. For example, in a 3-zone cluster, - MaxSkew is set to 1, and pods with the same - labelSelector spread as 3/1/1: | zone1 | zone2 - | zone3 | | P P P | P | P | If WhenUnsatisfiable - is set to DoNotSchedule, incoming pod can only - be scheduled to zone2(zone3) to become 3/2/1(3/1/2) - as ActualSkew(2-1) on zone2(zone3) satisfies - MaxSkew(1). In other words, the cluster can - still be imbalanced, but scheduler won''t make - it *more* imbalanced. It''s a required field.' - type: string - required: - - maxSkew - - topologyKey - - whenUnsatisfiable - type: object - type: array - x-kubernetes-list-map-keys: - - topologyKey - - whenUnsatisfiable - x-kubernetes-list-type: map - volumes: - description: 'List of volumes that can be mounted by - containers belonging to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes' - items: - description: Volume represents a named volume in a - pod that may be accessed by any container in the - pod. - properties: - awsElasticBlockStore: - description: 'AWSElasticBlockStore represents - an AWS Disk resource that is attached to a kubelet''s - host machine and then exposed to the pod. More - info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - properties: - fsType: - description: 'Filesystem type of the volume - that you want to mount. Tip: Ensure that - the filesystem type is supported by the - host operating system. Examples: "ext4", - "xfs", "ntfs". Implicitly inferred to be - "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - TODO: how do we prevent errors in the filesystem - from compromising the machine' - type: string - partition: - description: 'The partition in the volume - that you want to mount. If omitted, the - default is to mount by volume name. Examples: - For volume /dev/sda1, you specify the partition - as "1". Similarly, the volume partition - for /dev/sda is "0" (or you can leave the - property empty).' - format: int32 - type: integer - readOnly: - description: 'Specify "true" to force and - set the ReadOnly property in VolumeMounts - to "true". If omitted, the default is "false". - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - type: boolean - volumeID: - description: 'Unique ID of the persistent - disk resource in AWS (Amazon EBS volume). - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - type: string - required: - - volumeID - type: object - azureDisk: - description: AzureDisk represents an Azure Data - Disk mount on the host and bind mount to the - pod. - properties: - cachingMode: - description: 'Host Caching mode: None, Read - Only, Read Write.' - type: string - diskName: - description: The Name of the data disk in - the blob storage - type: string - diskURI: - description: The URI the data disk in the - blob storage - type: string - fsType: - description: Filesystem type to mount. Must - be a filesystem type supported by the host - operating system. Ex. "ext4", "xfs", "ntfs". - Implicitly inferred to be "ext4" if unspecified. - type: string - kind: - description: 'Expected values Shared: multiple - blob disks per storage account Dedicated: - single blob disk per storage account Managed: - azure managed data disk (only in managed - availability set). defaults to shared' - type: string - readOnly: - description: Defaults to false (read/write). - ReadOnly here will force the ReadOnly setting - in VolumeMounts. - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - description: AzureFile represents an Azure File - Service mount on the host and bind mount to - the pod. - properties: - readOnly: - description: Defaults to false (read/write). - ReadOnly here will force the ReadOnly setting - in VolumeMounts. - type: boolean - secretName: - description: the name of secret that contains - Azure Storage Account Name and Key - type: string - shareName: - description: Share Name - type: string - required: - - secretName - - shareName - type: object - cephfs: - description: CephFS represents a Ceph FS mount - on the host that shares a pod's lifetime - properties: - monitors: - description: 'Required: Monitors is a collection - of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - items: - type: string - type: array - path: - description: 'Optional: Used as the mounted - root, rather than the full Ceph tree, default - is /' - type: string - readOnly: - description: 'Optional: Defaults to false - (read/write). ReadOnly here will force the - ReadOnly setting in VolumeMounts. More info: - https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: boolean - secretFile: - description: 'Optional: SecretFile is the - path to key ring for User, default is /etc/ceph/user.secret - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: string - secretRef: - description: 'Optional: SecretRef is reference - to the authentication secret for User, default - is empty. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - properties: - name: - description: 'Name of the referent. More - info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - type: object - user: - description: 'Optional: User is the rados - user name, default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: string - required: - - monitors - type: object - cinder: - description: 'Cinder represents a cinder volume - attached and mounted on kubelets host machine. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - properties: - fsType: - description: 'Filesystem type to mount. Must - be a filesystem type supported by the host - operating system. Examples: "ext4", "xfs", - "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: string - readOnly: - description: 'Optional: Defaults to false - (read/write). ReadOnly here will force the - ReadOnly setting in VolumeMounts. More info: - https://examples.k8s.io/mysql-cinder-pd/README.md' - type: boolean - secretRef: - description: 'Optional: points to a secret - object containing parameters used to connect - to OpenStack.' - properties: - name: - description: 'Name of the referent. More - info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - type: object - volumeID: - description: 'volume id used to identify the - volume in cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: string - required: - - volumeID - type: object - configMap: - description: ConfigMap represents a configMap - that should populate this volume - properties: - defaultMode: - description: 'Optional: mode bits used to - set permissions on created files by default. - Must be an octal value between 0000 and - 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, - JSON requires decimal values for mode bits. - Defaults to 0644. Directories within the - path are not affected by this setting. This - might be in conflict with other options - that affect the file mode, like fsGroup, - and the result can be other mode bits set.' - format: int32 - type: integer - items: - description: If unspecified, each key-value - pair in the Data field of the referenced - ConfigMap will be projected into the volume - as a file whose name is the key and content - is the value. If specified, the listed keys - will be projected into the specified paths, - and unlisted keys will not be present. If - a key is specified which is not present - in the ConfigMap, the volume setup will - error unless it is marked optional. Paths - must be relative and may not contain the - '..' path or start with '..'. - items: - description: Maps a string key to a path - within a volume. - properties: - key: - description: The key to project. - type: string - mode: - description: 'Optional: mode bits used - to set permissions on this file. Must - be an octal value between 0000 and - 0777 or a decimal value between 0 - and 511. YAML accepts both octal and - decimal values, JSON requires decimal - values for mode bits. If not specified, - the volume defaultMode will be used. - This might be in conflict with other - options that affect the file mode, - like fsGroup, and the result can be - other mode bits set.' - format: int32 - type: integer - path: - description: The relative path of the - file to map the key to. May not be - an absolute path. May not contain - the path element '..'. May not start - with the string '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. More info: - https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - optional: - description: Specify whether the ConfigMap - or its keys must be defined - type: boolean - type: object - csi: - description: CSI (Container Storage Interface) - represents ephemeral storage that is handled - by certain external CSI drivers (Beta feature). - properties: - driver: - description: Driver is the name of the CSI - driver that handles this volume. Consult - with your admin for the correct name as - registered in the cluster. - type: string - fsType: - description: Filesystem type to mount. Ex. - "ext4", "xfs", "ntfs". If not provided, - the empty value is passed to the associated - CSI driver which will determine the default - filesystem to apply. - type: string - nodePublishSecretRef: - description: NodePublishSecretRef is a reference - to the secret object containing sensitive - information to pass to the CSI driver to - complete the CSI NodePublishVolume and NodeUnpublishVolume - calls. This field is optional, and may - be empty if no secret is required. If the - secret object contains more than one secret, - all secret references are passed. - properties: - name: - description: 'Name of the referent. More - info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - type: object - readOnly: - description: Specifies a read-only configuration - for the volume. Defaults to false (read/write). - type: boolean - volumeAttributes: - additionalProperties: - type: string - description: VolumeAttributes stores driver-specific - properties that are passed to the CSI driver. - Consult your driver's documentation for - supported values. - type: object - required: - - driver - type: object - downwardAPI: - description: DownwardAPI represents downward API - about the pod that should populate this volume - properties: - defaultMode: - description: 'Optional: mode bits to use on - created files by default. Must be a Optional: - mode bits used to set permissions on created - files by default. Must be an octal value - between 0000 and 0777 or a decimal value - between 0 and 511. YAML accepts both octal - and decimal values, JSON requires decimal - values for mode bits. Defaults to 0644. - Directories within the path are not affected - by this setting. This might be in conflict - with other options that affect the file - mode, like fsGroup, and the result can be - other mode bits set.' - format: int32 - type: integer - items: - description: Items is a list of downward API - volume file - items: - description: DownwardAPIVolumeFile represents - information to create the file containing - the pod field - properties: - fieldRef: - description: 'Required: Selects a field - of the pod: only annotations, labels, - name and namespace are supported.' - properties: - apiVersion: - description: Version of the schema - the FieldPath is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to - select in the specified API version. - type: string - required: - - fieldPath - type: object - mode: - description: 'Optional: mode bits used - to set permissions on this file, must - be an octal value between 0000 and - 0777 or a decimal value between 0 - and 511. YAML accepts both octal and - decimal values, JSON requires decimal - values for mode bits. If not specified, - the volume defaultMode will be used. - This might be in conflict with other - options that affect the file mode, - like fsGroup, and the result can be - other mode bits set.' - format: int32 - type: integer - path: - description: 'Required: Path is the - relative path name of the file to - be created. Must not be absolute or - contain the ''..'' path. Must be utf-8 - encoded. The first item of the relative - path must not start with ''..''' - type: string - resourceFieldRef: - description: 'Selects a resource of - the container: only resources limits - and requests (limits.cpu, limits.memory, - requests.cpu and requests.memory) - are currently supported.' - properties: - containerName: - description: 'Container name: required - for volumes, optional for env - vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource - to select' - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - emptyDir: - description: 'EmptyDir represents a temporary - directory that shares a pod''s lifetime. More - info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' - properties: - medium: - description: 'What type of storage medium - should back this directory. The default - is "" which means to use the node''s default - medium. Must be an empty string (default) - or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - description: 'Total amount of local storage - required for this EmptyDir volume. The size - limit is also applicable for memory medium. - The maximum usage on memory medium EmptyDir - would be the minimum value between the SizeLimit - specified here and the sum of memory limits - of all containers in a pod. The default - is nil which means that the limit is undefined. - More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: - description: "Ephemeral represents a volume that - is handled by a cluster storage driver (Alpha - feature). The volume's lifecycle is tied to - the pod that defines it - it will be created - before the pod starts, and deleted when the - pod is removed. \n Use this if: a) the volume - is only needed while the pod runs, b) features - of normal volumes like restoring from snapshot - or capacity tracking are needed, c) the storage - driver is specified through a storage class, - and d) the storage driver supports dynamic volume - provisioning through a PersistentVolumeClaim - (see EphemeralVolumeSource for more information - on the connection between this volume type and - PersistentVolumeClaim). \n Use PersistentVolumeClaim - or one of the vendor-specific APIs for volumes - that persist for longer than the lifecycle of - an individual pod. \n Use CSI for light-weight - local ephemeral volumes if the CSI driver is - meant to be used that way - see the documentation - of the driver for more information. \n A pod - can use both types of ephemeral volumes and - persistent volumes at the same time." - properties: - readOnly: - description: Specifies a read-only configuration - for the volume. Defaults to false (read/write). - type: boolean - volumeClaimTemplate: - description: "Will be used to create a stand-alone - PVC to provision the volume. The pod in - which this EphemeralVolumeSource is embedded - will be the owner of the PVC, i.e. the PVC - will be deleted together with the pod. The - name of the PVC will be `-` where `` is the name - from the `PodSpec.Volumes` array entry. - Pod validation will reject the pod if the - concatenated name is not valid for a PVC - (for example, too long). \n An existing - PVC with that name that is not owned by - the pod will *not* be used for the pod to - avoid using an unrelated volume by mistake. - Starting the pod is then blocked until the - unrelated PVC is removed. If such a pre-created - PVC is meant to be used by the pod, the - PVC has to updated with an owner reference - to the pod once the pod exists. Normally - this should not be necessary, but it may - be useful when manually reconstructing a - broken cluster. \n This field is read-only - and no changes will be made by Kubernetes - to the PVC after it has been created. \n - Required, must not be nil." - properties: - metadata: - description: May contain labels and annotations - that will be copied into the PVC when - creating it. No other fields are allowed - and will be rejected during validation. - type: object - spec: - description: The specification for the - PersistentVolumeClaim. The entire content - is copied unchanged into the PVC that - gets created from this template. The - same fields as in a PersistentVolumeClaim - are also valid here. - properties: - accessModes: - description: 'AccessModes contains - the desired access modes the volume - should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' - items: - type: string - type: array - dataSource: - description: 'This field can be used - to specify either: * An existing - VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) - * An existing custom resource that - implements data population (Alpha) - In order to use custom resource - types that implement data population, - the AnyVolumeDataSource feature - gate must be enabled. If the provisioner - or an external controller can support - the specified data source, it will - create a new volume based on the - contents of the specified data source.' - properties: - apiGroup: - description: APIGroup is the group - for the resource being referenced. - If APIGroup is not specified, - the specified Kind must be in - the core API group. For any - other third-party types, APIGroup - is required. - type: string - kind: - description: Kind is the type - of resource being referenced - type: string - name: - description: Name is the name - of resource being referenced - type: string - required: - - kind - - name - type: object - resources: - description: 'Resources represents - the minimum resources the volume - should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Limits describes - the maximum amount of compute - resources allowed. More info: - https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes - the minimum amount of compute - resources required. If Requests - is omitted for a container, - it defaults to Limits if that - is explicitly specified, otherwise - to an implementation-defined - value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' - type: object - type: object - selector: - description: A label query over volumes - to consider for binding. - properties: - matchExpressions: - description: matchExpressions - is a list of label selector - requirements. The requirements - are ANDed. - items: - description: A label selector - requirement is a selector - that contains values, a key, - and an operator that relates - the key and values. - properties: - key: - description: key is the - label key that the selector - applies to. - type: string - operator: - description: operator represents - a key's relationship to - a set of values. Valid - operators are In, NotIn, - Exists and DoesNotExist. - type: string - values: - description: values is an - array of string values. - If the operator is In - or NotIn, the values array - must be non-empty. If - the operator is Exists - or DoesNotExist, the values - array must be empty. This - array is replaced during - a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a - map of {key,value} pairs. A - single {key,value} in the matchLabels - map is equivalent to an element - of matchExpressions, whose key - field is "key", the operator - is "In", and the values array - contains only "value". The requirements - are ANDed. - type: object - type: object - storageClassName: - description: 'Name of the StorageClass - required by the claim. More info: - https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' - type: string - volumeMode: - description: volumeMode defines what - type of volume is required by the - claim. Value of Filesystem is implied - when not included in claim spec. - type: string - volumeName: - description: VolumeName is the binding - reference to the PersistentVolume - backing this claim. - type: string - type: object - required: - - spec - type: object - type: object - fc: - description: FC represents a Fibre Channel resource - that is attached to a kubelet's host machine - and then exposed to the pod. - properties: - fsType: - description: 'Filesystem type to mount. Must - be a filesystem type supported by the host - operating system. Ex. "ext4", "xfs", "ntfs". - Implicitly inferred to be "ext4" if unspecified. - TODO: how do we prevent errors in the filesystem - from compromising the machine' - type: string - lun: - description: 'Optional: FC target lun number' - format: int32 - type: integer - readOnly: - description: 'Optional: Defaults to false - (read/write). ReadOnly here will force the - ReadOnly setting in VolumeMounts.' - type: boolean - targetWWNs: - description: 'Optional: FC target worldwide - names (WWNs)' - items: - type: string - type: array - wwids: - description: 'Optional: FC volume world wide - identifiers (wwids) Either wwids or combination - of targetWWNs and lun must be set, but not - both simultaneously.' - items: - type: string - type: array - type: object - flexVolume: - description: FlexVolume represents a generic volume - resource that is provisioned/attached using - an exec based plugin. - properties: - driver: - description: Driver is the name of the driver - to use for this volume. - type: string - fsType: - description: Filesystem type to mount. Must - be a filesystem type supported by the host - operating system. Ex. "ext4", "xfs", "ntfs". - The default filesystem depends on FlexVolume - script. - type: string - options: - additionalProperties: - type: string - description: 'Optional: Extra command options - if any.' - type: object - readOnly: - description: 'Optional: Defaults to false - (read/write). ReadOnly here will force the - ReadOnly setting in VolumeMounts.' - type: boolean - secretRef: - description: 'Optional: SecretRef is reference - to the secret object containing sensitive - information to pass to the plugin scripts. - This may be empty if no secret object is - specified. If the secret object contains - more than one secret, all secrets are passed - to the plugin scripts.' - properties: - name: - description: 'Name of the referent. More - info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - type: object - required: - - driver - type: object - flocker: - description: Flocker represents a Flocker volume - attached to a kubelet's host machine. This depends - on the Flocker control service being running - properties: - datasetName: - description: Name of the dataset stored as - metadata -> name on the dataset for Flocker - should be considered as deprecated - type: string - datasetUUID: - description: UUID of the dataset. This is - unique identifier of a Flocker dataset - type: string - type: object - gcePersistentDisk: - description: 'GCEPersistentDisk represents a GCE - Disk resource that is attached to a kubelet''s - host machine and then exposed to the pod. More - info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - properties: - fsType: - description: 'Filesystem type of the volume - that you want to mount. Tip: Ensure that - the filesystem type is supported by the - host operating system. Examples: "ext4", - "xfs", "ntfs". Implicitly inferred to be - "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - TODO: how do we prevent errors in the filesystem - from compromising the machine' - type: string - partition: - description: 'The partition in the volume - that you want to mount. If omitted, the - default is to mount by volume name. Examples: - For volume /dev/sda1, you specify the partition - as "1". Similarly, the volume partition - for /dev/sda is "0" (or you can leave the - property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - format: int32 - type: integer - pdName: - description: 'Unique name of the PD resource - in GCE. Used to identify the disk in GCE. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - type: string - readOnly: - description: 'ReadOnly here will force the - ReadOnly setting in VolumeMounts. Defaults - to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - type: boolean - required: - - pdName - type: object - gitRepo: - description: 'GitRepo represents a git repository - at a particular revision. DEPRECATED: GitRepo - is deprecated. To provision a container with - a git repo, mount an EmptyDir into an InitContainer - that clones the repo using git, then mount the - EmptyDir into the Pod''s container.' - properties: - directory: - description: Target directory name. Must not - contain or start with '..'. If '.' is supplied, - the volume directory will be the git repository. Otherwise, - if specified, the volume will contain the - git repository in the subdirectory with - the given name. - type: string - repository: - description: Repository URL - type: string - revision: - description: Commit hash for the specified - revision. - type: string - required: - - repository - type: object - glusterfs: - description: 'Glusterfs represents a Glusterfs - mount on the host that shares a pod''s lifetime. - More info: https://examples.k8s.io/volumes/glusterfs/README.md' - properties: - endpoints: - description: 'EndpointsName is the endpoint - name that details Glusterfs topology. More - info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' - type: string - path: - description: 'Path is the Glusterfs volume - path. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' - type: string - readOnly: - description: 'ReadOnly here will force the - Glusterfs volume to be mounted with read-only - permissions. Defaults to false. More info: - https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' - type: boolean - required: - - endpoints - - path - type: object - hostPath: - description: 'HostPath represents a pre-existing - file or directory on the host machine that is - directly exposed to the container. This is generally - used for system agents or other privileged things - that are allowed to see the host machine. Most - containers will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - --- TODO(jonesdl) We need to restrict who can - use host directory mounts and who can/can not - mount host directories as read/write.' - properties: - path: - description: 'Path of the directory on the - host. If the path is a symlink, it will - follow the link to the real path. More info: - https://kubernetes.io/docs/concepts/storage/volumes#hostpath' - type: string - type: - description: 'Type for HostPath Volume Defaults - to "" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' - type: string - required: - - path - type: object - iscsi: - description: 'ISCSI represents an ISCSI Disk resource - that is attached to a kubelet''s host machine - and then exposed to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' - properties: - chapAuthDiscovery: - description: whether support iSCSI Discovery - CHAP authentication - type: boolean - chapAuthSession: - description: whether support iSCSI Session - CHAP authentication - type: boolean - fsType: - description: 'Filesystem type of the volume - that you want to mount. Tip: Ensure that - the filesystem type is supported by the - host operating system. Examples: "ext4", - "xfs", "ntfs". Implicitly inferred to be - "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi - TODO: how do we prevent errors in the filesystem - from compromising the machine' - type: string - initiatorName: - description: Custom iSCSI Initiator Name. - If initiatorName is specified with iscsiInterface - simultaneously, new iSCSI interface : will be created for - the connection. - type: string - iqn: - description: Target iSCSI Qualified Name. - type: string - iscsiInterface: - description: iSCSI Interface Name that uses - an iSCSI transport. Defaults to 'default' - (tcp). - type: string - lun: - description: iSCSI Target Lun number. - format: int32 - type: integer - portals: - description: iSCSI Target Portal List. The - portal is either an IP or ip_addr:port if - the port is other than default (typically - TCP ports 860 and 3260). - items: - type: string - type: array - readOnly: - description: ReadOnly here will force the - ReadOnly setting in VolumeMounts. Defaults - to false. - type: boolean - secretRef: - description: CHAP Secret for iSCSI target - and initiator authentication - properties: - name: - description: 'Name of the referent. More - info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - type: object - targetPortal: - description: iSCSI Target Portal. The Portal - is either an IP or ip_addr:port if the port - is other than default (typically TCP ports - 860 and 3260). - type: string - required: - - iqn - - lun - - targetPortal - type: object - name: - description: 'Volume''s name. Must be a DNS_LABEL - and unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - nfs: - description: 'NFS represents an NFS mount on the - host that shares a pod''s lifetime More info: - https://kubernetes.io/docs/concepts/storage/volumes#nfs' - properties: - path: - description: 'Path that is exported by the - NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: string - readOnly: - description: 'ReadOnly here will force the - NFS export to be mounted with read-only - permissions. Defaults to false. More info: - https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: boolean - server: - description: 'Server is the hostname or IP - address of the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - description: 'PersistentVolumeClaimVolumeSource - represents a reference to a PersistentVolumeClaim - in the same namespace. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' - properties: - claimName: - description: 'ClaimName is the name of a PersistentVolumeClaim - in the same namespace as the pod using this - volume. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' - type: string - readOnly: - description: Will force the ReadOnly setting - in VolumeMounts. Default false. - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - description: PhotonPersistentDisk represents a - PhotonController persistent disk attached and - mounted on kubelets host machine - properties: - fsType: - description: Filesystem type to mount. Must - be a filesystem type supported by the host - operating system. Ex. "ext4", "xfs", "ntfs". - Implicitly inferred to be "ext4" if unspecified. - type: string - pdID: - description: ID that identifies Photon Controller - persistent disk - type: string - required: - - pdID - type: object - portworxVolume: - description: PortworxVolume represents a portworx - volume attached and mounted on kubelets host - machine - properties: - fsType: - description: FSType represents the filesystem - type to mount Must be a filesystem type - supported by the host operating system. - Ex. "ext4", "xfs". Implicitly inferred to - be "ext4" if unspecified. - type: string - readOnly: - description: Defaults to false (read/write). - ReadOnly here will force the ReadOnly setting - in VolumeMounts. - type: boolean - volumeID: - description: VolumeID uniquely identifies - a Portworx volume - type: string - required: - - volumeID - type: object - projected: - description: Items for all in one resources secrets, - configmaps, and downward API - properties: - defaultMode: - description: Mode bits used to set permissions - on created files by default. Must be an - octal value between 0000 and 0777 or a decimal - value between 0 and 511. YAML accepts both - octal and decimal values, JSON requires - decimal values for mode bits. Directories - within the path are not affected by this - setting. This might be in conflict with - other options that affect the file mode, - like fsGroup, and the result can be other - mode bits set. - format: int32 - type: integer - sources: - description: list of volume projections - items: - description: Projection that may be projected - along with other supported volume types - properties: - configMap: - description: information about the configMap - data to project - properties: - items: - description: If unspecified, each - key-value pair in the Data field - of the referenced ConfigMap will - be projected into the volume as - a file whose name is the key and - content is the value. If specified, - the listed keys will be projected - into the specified paths, and - unlisted keys will not be present. - If a key is specified which is - not present in the ConfigMap, - the volume setup will error unless - it is marked optional. Paths must - be relative and may not contain - the '..' path or start with '..'. - items: - description: Maps a string key - to a path within a volume. - properties: - key: - description: The key to project. - type: string - mode: - description: 'Optional: mode - bits used to set permissions - on this file. Must be an - octal value between 0000 - and 0777 or a decimal value - between 0 and 511. YAML - accepts both octal and decimal - values, JSON requires decimal - values for mode bits. If - not specified, the volume - defaultMode will be used. - This might be in conflict - with other options that - affect the file mode, like - fsGroup, and the result - can be other mode bits set.' - format: int32 - type: integer - path: - description: The relative - path of the file to map - the key to. May not be an - absolute path. May not contain - the path element '..'. May - not start with the string - '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. - apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the - ConfigMap or its keys must be - defined - type: boolean - type: object - downwardAPI: - description: information about the downwardAPI - data to project - properties: - items: - description: Items is a list of - DownwardAPIVolume file - items: - description: DownwardAPIVolumeFile - represents information to create - the file containing the pod - field - properties: - fieldRef: - description: 'Required: Selects - a field of the pod: only - annotations, labels, name - and namespace are supported.' - properties: - apiVersion: - description: Version of - the schema the FieldPath - is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the - field to select in the - specified API version. - type: string - required: - - fieldPath - type: object - mode: - description: 'Optional: mode - bits used to set permissions - on this file, must be an - octal value between 0000 - and 0777 or a decimal value - between 0 and 511. YAML - accepts both octal and decimal - values, JSON requires decimal - values for mode bits. If - not specified, the volume - defaultMode will be used. - This might be in conflict - with other options that - affect the file mode, like - fsGroup, and the result - can be other mode bits set.' - format: int32 - type: integer - path: - description: 'Required: Path - is the relative path name - of the file to be created. - Must not be absolute or - contain the ''..'' path. - Must be utf-8 encoded. The - first item of the relative - path must not start with - ''..''' - type: string - resourceFieldRef: - description: 'Selects a resource - of the container: only resources - limits and requests (limits.cpu, - limits.memory, requests.cpu - and requests.memory) are - currently supported.' - properties: - containerName: - description: 'Container - name: required for volumes, - optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies - the output format of - the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: - resource to select' - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - secret: - description: information about the secret - data to project - properties: - items: - description: If unspecified, each - key-value pair in the Data field - of the referenced Secret will - be projected into the volume as - a file whose name is the key and - content is the value. If specified, - the listed keys will be projected - into the specified paths, and - unlisted keys will not be present. - If a key is specified which is - not present in the Secret, the - volume setup will error unless - it is marked optional. Paths must - be relative and may not contain - the '..' path or start with '..'. - items: - description: Maps a string key - to a path within a volume. - properties: - key: - description: The key to project. - type: string - mode: - description: 'Optional: mode - bits used to set permissions - on this file. Must be an - octal value between 0000 - and 0777 or a decimal value - between 0 and 511. YAML - accepts both octal and decimal - values, JSON requires decimal - values for mode bits. If - not specified, the volume - defaultMode will be used. - This might be in conflict - with other options that - affect the file mode, like - fsGroup, and the result - can be other mode bits set.' - format: int32 - type: integer - path: - description: The relative - path of the file to map - the key to. May not be an - absolute path. May not contain - the path element '..'. May - not start with the string - '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. - apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the - Secret or its key must be defined - type: boolean - type: object - serviceAccountToken: - description: information about the serviceAccountToken - data to project - properties: - audience: - description: Audience is the intended - audience of the token. A recipient - of a token must identify itself - with an identifier specified in - the audience of the token, and - otherwise should reject the token. - The audience defaults to the identifier - of the apiserver. - type: string - expirationSeconds: - description: ExpirationSeconds is - the requested duration of validity - of the service account token. - As the token approaches expiration, - the kubelet volume plugin will - proactively rotate the service - account token. The kubelet will - start trying to rotate the token - if the token is older than 80 - percent of its time to live or - if the token is older than 24 - hours.Defaults to 1 hour and must - be at least 10 minutes. - format: int64 - type: integer - path: - description: Path is the path relative - to the mount point of the file - to project the token into. - type: string - required: - - path - type: object - type: object - type: array - type: object - quobyte: - description: Quobyte represents a Quobyte mount - on the host that shares a pod's lifetime - properties: - group: - description: Group to map volume access to - Default is no group - type: string - readOnly: - description: ReadOnly here will force the - Quobyte volume to be mounted with read-only - permissions. Defaults to false. - type: boolean - registry: - description: Registry represents a single - or multiple Quobyte Registry services specified - as a string as host:port pair (multiple - entries are separated with commas) which - acts as the central registry for volumes - type: string - tenant: - description: Tenant owning the given Quobyte - volume in the Backend Used with dynamically - provisioned Quobyte volumes, value is set - by the plugin - type: string - user: - description: User to map volume access to - Defaults to serivceaccount user - type: string - volume: - description: Volume is a string that references - an already created Quobyte volume by name. - type: string - required: - - registry - - volume - type: object - rbd: - description: 'RBD represents a Rados Block Device - mount on the host that shares a pod''s lifetime. - More info: https://examples.k8s.io/volumes/rbd/README.md' - properties: - fsType: - description: 'Filesystem type of the volume - that you want to mount. Tip: Ensure that - the filesystem type is supported by the - host operating system. Examples: "ext4", - "xfs", "ntfs". Implicitly inferred to be - "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd - TODO: how do we prevent errors in the filesystem - from compromising the machine' - type: string - image: - description: 'The rados image name. More info: - https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - keyring: - description: 'Keyring is the path to key ring - for RBDUser. Default is /etc/ceph/keyring. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - monitors: - description: 'A collection of Ceph monitors. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - items: - type: string - type: array - pool: - description: 'The rados pool name. Default - is rbd. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - readOnly: - description: 'ReadOnly here will force the - ReadOnly setting in VolumeMounts. Defaults - to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: boolean - secretRef: - description: 'SecretRef is name of the authentication - secret for RBDUser. If provided overrides - keyring. Default is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - properties: - name: - description: 'Name of the referent. More - info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - type: object - user: - description: 'The rados user name. Default - is admin. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - required: - - image - - monitors - type: object - scaleIO: - description: ScaleIO represents a ScaleIO persistent - volume attached and mounted on Kubernetes nodes. - properties: - fsType: - description: Filesystem type to mount. Must - be a filesystem type supported by the host - operating system. Ex. "ext4", "xfs", "ntfs". - Default is "xfs". - type: string - gateway: - description: The host address of the ScaleIO - API Gateway. - type: string - protectionDomain: - description: The name of the ScaleIO Protection - Domain for the configured storage. - type: string - readOnly: - description: Defaults to false (read/write). - ReadOnly here will force the ReadOnly setting - in VolumeMounts. - type: boolean - secretRef: - description: SecretRef references to the secret - for ScaleIO user and other sensitive information. - If this is not provided, Login operation - will fail. - properties: - name: - description: 'Name of the referent. More - info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - type: object - sslEnabled: - description: Flag to enable/disable SSL communication - with Gateway, default false - type: boolean - storageMode: - description: Indicates whether the storage - for a volume should be ThickProvisioned - or ThinProvisioned. Default is ThinProvisioned. - type: string - storagePool: - description: The ScaleIO Storage Pool associated - with the protection domain. - type: string - system: - description: The name of the storage system - as configured in ScaleIO. - type: string - volumeName: - description: The name of a volume already - created in the ScaleIO system that is associated - with this volume source. - type: string - required: - - gateway - - secretRef - - system - type: object - secret: - description: 'Secret represents a secret that - should populate this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' - properties: - defaultMode: - description: 'Optional: mode bits used to - set permissions on created files by default. - Must be an octal value between 0000 and - 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, - JSON requires decimal values for mode bits. - Defaults to 0644. Directories within the - path are not affected by this setting. This - might be in conflict with other options - that affect the file mode, like fsGroup, - and the result can be other mode bits set.' - format: int32 - type: integer - items: - description: If unspecified, each key-value - pair in the Data field of the referenced - Secret will be projected into the volume - as a file whose name is the key and content - is the value. If specified, the listed keys - will be projected into the specified paths, - and unlisted keys will not be present. If - a key is specified which is not present - in the Secret, the volume setup will error - unless it is marked optional. Paths must - be relative and may not contain the '..' - path or start with '..'. - items: - description: Maps a string key to a path - within a volume. - properties: - key: - description: The key to project. - type: string - mode: - description: 'Optional: mode bits used - to set permissions on this file. Must - be an octal value between 0000 and - 0777 or a decimal value between 0 - and 511. YAML accepts both octal and - decimal values, JSON requires decimal - values for mode bits. If not specified, - the volume defaultMode will be used. - This might be in conflict with other - options that affect the file mode, - like fsGroup, and the result can be - other mode bits set.' - format: int32 - type: integer - path: - description: The relative path of the - file to map the key to. May not be - an absolute path. May not contain - the path element '..'. May not start - with the string '..'. - type: string - required: - - key - - path - type: object - type: array - optional: - description: Specify whether the Secret or - its keys must be defined - type: boolean - secretName: - description: 'Name of the secret in the pod''s - namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' - type: string - type: object - storageos: - description: StorageOS represents a StorageOS - volume attached and mounted on Kubernetes nodes. - properties: - fsType: - description: Filesystem type to mount. Must - be a filesystem type supported by the host - operating system. Ex. "ext4", "xfs", "ntfs". - Implicitly inferred to be "ext4" if unspecified. - type: string - readOnly: - description: Defaults to false (read/write). - ReadOnly here will force the ReadOnly setting - in VolumeMounts. - type: boolean - secretRef: - description: SecretRef specifies the secret - to use for obtaining the StorageOS API credentials. If - not specified, default values will be attempted. - properties: - name: - description: 'Name of the referent. More - info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - type: object - volumeName: - description: VolumeName is the human-readable - name of the StorageOS volume. Volume names - are only unique within a namespace. - type: string - volumeNamespace: - description: VolumeNamespace specifies the - scope of the volume within StorageOS. If - no namespace is specified then the Pod's - namespace will be used. This allows the - Kubernetes name scoping to be mirrored within - StorageOS for tighter integration. Set VolumeName - to any name to override the default behaviour. - Set to "default" if you are not using namespaces - within StorageOS. Namespaces that do not - pre-exist within StorageOS will be created. - type: string - type: object - vsphereVolume: - description: VsphereVolume represents a vSphere - volume attached and mounted on kubelets host - machine - properties: - fsType: - description: Filesystem type to mount. Must - be a filesystem type supported by the host - operating system. Ex. "ext4", "xfs", "ntfs". - Implicitly inferred to be "ext4" if unspecified. - type: string - storagePolicyID: - description: Storage Policy Based Management - (SPBM) profile ID associated with the StoragePolicyName. - type: string - storagePolicyName: - description: Storage Policy Based Management - (SPBM) profile name. - type: string - volumePath: - description: Path that identifies vSphere - volume vmdk - type: string - required: - - volumePath - type: object - required: - - name - type: object - type: array - required: - - containers - type: object - type: object - type: object - type: array - type: object - status: - description: Current status of QueueJob - properties: - Succeeded: - description: The number of pods which reached phase Succeeded. - format: int32 - type: integer - failed: - description: The number of pods which reached phase Failed. - format: int32 - type: integer - minAvailable: - description: The minimal available pods to run for this QueueJob - format: int32 - type: integer - pending: - description: The number of pending pods. - format: int32 - type: integer - running: - description: The number of running pods. - format: int32 - type: integer - type: object - type: object - served: true - storage: true diff --git a/config/crd/bases/mcad.ibm.com_schedulingspecs.yaml b/config/crd/bases/mcad.ibm.com_schedulingspecs.yaml deleted file mode 100644 index e6655b869..000000000 --- a/config/crd/bases/mcad.ibm.com_schedulingspecs.yaml +++ /dev/null @@ -1,68 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.4.1 - creationTimestamp: null - name: schedulingspecs.mcad.ibm.com -spec: - group: mcad.ibm.com - names: - kind: SchedulingSpec - listKind: SchedulingSpecList - plural: schedulingspecs - singular: schedulingspec - scope: Namespaced - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - properties: - minAvailable: - type: integer - requeuing: - description: Specification of the requeuing strategy based on - waiting time - properties: - initialTimeInSeconds: - type: integer - timeInSeconds: - type: integer - default: 300 - maxTimeInSeconds: - type: integer - default: 0 - growthType: - type: string - default: "exponential" - numRequeuings: - type: integer - default: 0 - maxNumRequeuings: - type: integer - default: 0 - type: object - nodeSelector: - additionalProperties: - type: string - type: object - type: object - required: - - metadata - type: object - served: true - storage: true diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 1b1d7b0b8..725db0122 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -4,9 +4,7 @@ resources: - bases/codeflare.codeflare.dev_mcads.yaml - bases/codeflare.codeflare.dev_instascales.yaml -- bases/mcad.ibm.com_appwrappers.yaml -- bases/mcad.ibm.com_queuejobs.yaml -- bases/mcad.ibm.com_schedulingspecs.yaml +- mcad #+kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: diff --git a/config/crd/mcad/kustomization.yaml b/config/crd/mcad/kustomization.yaml new file mode 100644 index 000000000..d87d2f570 --- /dev/null +++ b/config/crd/mcad/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=main From f228dbadbcb7d69ed83837c3f9264e1e3bc7c3c4 Mon Sep 17 00:00:00 2001 From: Kevin Date: Wed, 24 May 2023 17:47:00 -0400 Subject: [PATCH 003/100] add new github action which tags and builds a new operator Signed-off-by: Kevin --- .github/workflows/tag-and-build.yml | 70 +++++++++++++++++++++++++++++ Makefile | 7 ++- 2 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/tag-and-build.yml diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml new file mode 100644 index 000000000..38fd559e4 --- /dev/null +++ b/.github/workflows/tag-and-build.yml @@ -0,0 +1,70 @@ +# This workflow will build the CodeFlare Operator image and push it to the project-codeflare image registry + +name: Tag and Release +on: + workflow_dispatch: + inputs: + version: + description: 'Tag to be used for operator image' + required: true + default: '0.0.0-dev' + replaces: + description: 'The previous semantic version that this tag replaces.' + required: true + default: '0.0.0-dev' + +jobs: + push: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Activate cache + uses: actions/cache@v2 + with: + path: /cache + key: ${{ runner.os }}-cache-${{ hashFiles('**/go.sum', '.pre-commit-config.yaml') }} + + - name: Create tag + uses: actions/github-script@v6 + with: + script: | + github.rest.git.createRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: 'refs/tags/${{ github.event.inputs.version }}', + sha: context.sha + }) + + - name: Install operator-sdk + run: make install-operator-sdk + + - name: Login to Quay.io + uses: redhat-actions/podman-login@v1 + with: + username: ${{ secrets.QUAY_ID }} + password: ${{ secrets.QUAY_TOKEN }} + registry: quay.io + + - name: Login to Red Hat Registry + uses: redhat-actions/podman-login@v1 + with: + username: ${{ secrets.RH_REG_ID }} + password: ${{ secrets.RH_REG_TOKEN }} + registry: registry.redhat.io + + - name: Image Build + run: | + make build + make bundle + make image-build -e IMG=quay.io/project-codeflare/codeflare-operator:${SOURCE_TAG} + podman tag quay.io/project-codeflare/codeflare-operator:${SOURCE_TAG} quay.io/project-codeflare/codeflare-operator:latest + env: + SOURCE_TAG: ${{ github.event.inputs.version }} + + - name: Image Push + run: | + make image-push -e IMG=quay.io/project-codeflare/codeflare-operator:${SOURCE_TAG} + make image-push -e IMG=quay.io/project-codeflare/codeflare-operator:latest + env: + SOURCE_TAG: ${{ github.event.inputs.version }} diff --git a/Makefile b/Makefile index 182059677..9bad4f544 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,11 @@ # To re-generate a bundle for another specific version without changing the standard setup, you can: # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) # - use environment variables to overwrite this value (e.g export VERSION=0.0.2) -VERSION ?= 0.0.2 +# best if we could detect this. If we cannot, we need to document it somewhere. +# then we can add a patch in the `PHONY: bundle` + +PREVIOUS_VERSION ?= 0.0.3 +VERSION ?= 0.0.3-dev # CHANNELS define the bundle channels used in the bundle. # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") @@ -193,6 +197,7 @@ bundle: manifests kustomize install-operator-sdk ## Generate bundle manifests an $(OPERATOR_SDK) generate kustomize manifests -q cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) cd config/manifests && $(KUSTOMIZE) edit add patch --patch '[{"op":"add", "path":"/metadata/annotations/containerImage", "value": "$(IMG)" }]' --kind ClusterServiceVersion + cd config/manifests && $(KUSTOMIZE) edit add patch --patch '[{"op":"add", "path":"/spec/replaces", "value": "codeflare-operator.v$(PREVIOUS_VERSION)" }]' --kind ClusterServiceVersion $(KUSTOMIZE) build config/manifests | $(OPERATOR_SDK) generate bundle $(BUNDLE_GEN_FLAGS) $(MAKE) validate-bundle From be426930e804e14053457b9529cc14d16e796164 Mon Sep 17 00:00:00 2001 From: Kevin Date: Thu, 1 Jun 2023 09:31:56 -0400 Subject: [PATCH 004/100] use unauthenticated red hat registry for Docker file this makes the images easier to build for those outside of the red hat organization Signed-off-by: Kevin --- .github/workflows/operator-image.yml | 7 ------- .github/workflows/tag-and-build.yml | 7 ------- Dockerfile | 2 +- 3 files changed, 1 insertion(+), 15 deletions(-) diff --git a/.github/workflows/operator-image.yml b/.github/workflows/operator-image.yml index 85d0a9ad6..54c6403ab 100644 --- a/.github/workflows/operator-image.yml +++ b/.github/workflows/operator-image.yml @@ -31,13 +31,6 @@ jobs: password: ${{ secrets.QUAY_TOKEN }} registry: quay.io - - name: Login to Red Hat Registry - uses: redhat-actions/podman-login@v1 - with: - username: ${{ secrets.RH_REG_ID }} - password: ${{ secrets.RH_REG_TOKEN }} - registry: registry.redhat.io - - name: Image Build run: | make build diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 38fd559e4..28653862d 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -46,13 +46,6 @@ jobs: password: ${{ secrets.QUAY_TOKEN }} registry: quay.io - - name: Login to Red Hat Registry - uses: redhat-actions/podman-login@v1 - with: - username: ${{ secrets.RH_REG_ID }} - password: ${{ secrets.RH_REG_TOKEN }} - registry: registry.redhat.io - - name: Image Build run: | make build diff --git a/Dockerfile b/Dockerfile index 8f465f673..b97173679 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM registry.redhat.io/ubi8/go-toolset:1.18.9-8 as builder +FROM registry.access.redhat.com/ubi8/go-toolset:1.18.9-8 as builder WORKDIR /workspace # Copy the Go Modules manifests From 15df44b84e21c3a9fa4ed5f94be50634fca2e0dd Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 1 Jun 2023 15:53:38 +0200 Subject: [PATCH 005/100] feat(api): Support custom InstaScale container image --- Makefile | 38 ++++++++++++++++--- api/v1alpha1/instascale_types.go | 17 +++++++-- .../codeflare.codeflare.dev_instascales.yaml | 11 ++++++ .../internal/instascale/deployment.yaml.tmpl | 2 +- controllers/defaults.go | 9 +++++ controllers/instascale_controller.go | 38 +++++++++---------- controllers/instascale_params.go | 18 ++++++--- 7 files changed, 99 insertions(+), 34 deletions(-) create mode 100644 controllers/defaults.go diff --git a/Makefile b/Makefile index 9bad4f544..a6e5e6bae 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,9 @@ PREVIOUS_VERSION ?= 0.0.3 VERSION ?= 0.0.3-dev +# INSTASCALE_VERSION defines the default version of the InstaScale controller +INSTASCALE_VERSION ?= v0.0.3 + # CHANNELS define the bundle channels used in the bundle. # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") # To re-generate a bundle for other specific channels without changing the standard setup, you can: @@ -28,12 +31,18 @@ BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL) endif BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) +# IMAGE_ORG_BASE defines the base container registry and organization for container images. +IMAGE_ORG_BASE ?= quay.io/project-codeflare + # IMAGE_TAG_BASE defines the docker.io namespace and part of the image name for remote images. # This variable is used to construct full image tags for bundle and catalog images. # # For example, running 'make bundle-build bundle-push catalog-build catalog-push' will build and push both # codeflare.dev/codeflare-operator-bundle:$VERSION and codeflare.dev/codeflare-operator-catalog:$VERSION. -IMAGE_TAG_BASE ?= quay.io/project-codeflare/codeflare-operator +IMAGE_TAG_BASE ?= $(IMAGE_ORG_BASE)/codeflare-operator + +# INSTASCALE_IMAGE defines the default container image for the InstaScale controller +INSTASCALE_IMAGE ?= $(IMAGE_ORG_BASE)/instascale-controller:$(INSTASCALE_VERSION) # BUNDLE_IMG defines the image:tag used for the bundle. # You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) @@ -89,6 +98,25 @@ help: ## Display this help. ##@ Development +DEFAULTS_FILE := controllers/defaults.go + +.PHONY: defaults +defaults: + $(info Regenerating $(DEFAULTS_FILE)) + @echo "package controllers" > $(DEFAULTS_FILE) + @echo "" >> $(DEFAULTS_FILE) + @echo "// ***********************" >> $(DEFAULTS_FILE) + @echo "// DO NOT EDIT THIS FILE" >> $(DEFAULTS_FILE) + @echo "// ***********************" >> $(DEFAULTS_FILE) + @echo "" >> $(DEFAULTS_FILE) + @echo "const (" >> $(DEFAULTS_FILE) + @echo " InstaScaleImage = \"$(INSTASCALE_IMAGE)\"" >> $(DEFAULTS_FILE) + @echo "" >> $(DEFAULTS_FILE) + @echo ")" >> $(DEFAULTS_FILE) + @echo "" >> $(DEFAULTS_FILE) + + gofmt -w $(DEFAULTS_FILE) + .PHONY: manifests manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases @@ -109,11 +137,11 @@ vet: ## Run go vet against code. ##@ Build .PHONY: build -build: generate fmt vet ## Build manager binary. +build: defaults generate fmt vet ## Build manager binary. go build -o bin/manager main.go .PHONY: run -run: manifests generate fmt vet ## Run a controller from your host. +run: defaults manifests generate fmt vet ## Run a controller from your host. go run ./main.go .PHONY: image-build @@ -193,7 +221,7 @@ validate-bundle: install-operator-sdk $(OPERATOR_SDK) bundle validate ./bundle --select-optional suite=operatorframework .PHONY: bundle -bundle: manifests kustomize install-operator-sdk ## Generate bundle manifests and metadata, then validate generated files. +bundle: defaults manifests kustomize install-operator-sdk ## Generate bundle manifests and metadata, then validate generated files. $(OPERATOR_SDK) generate kustomize manifests -q cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) cd config/manifests && $(KUSTOMIZE) edit add patch --patch '[{"op":"add", "path":"/metadata/annotations/containerImage", "value": "$(IMG)" }]' --kind ClusterServiceVersion @@ -251,5 +279,5 @@ catalog-push: ## Push a catalog image. $(MAKE) image-push IMG=$(CATALOG_IMG) .PHONY: test-unit -test-unit: manifests generate fmt vet envtest ## Run tests. +test-unit: defaults manifests generate fmt vet envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -coverprofile cover.out diff --git a/api/v1alpha1/instascale_types.go b/api/v1alpha1/instascale_types.go index b361654b6..063f9a0d4 100644 --- a/api/v1alpha1/instascale_types.go +++ b/api/v1alpha1/instascale_types.go @@ -38,6 +38,17 @@ type InstaScaleSpec struct { // controllerResources determines the container resources for the InstaScale controller deployment ControllerResources *v1.ResourceRequirements `json:"controllerResources,omitempty"` + + // The container image for the InstaScale controller deployment. + // If specified, the provided container image must be compatible with the running CodeFlare operator. + // Using an incompatible, or unrelated container image, will result in an undefined behavior. + // A CodeFlare operator upgrade will not upgrade the InstaScale controller, that'll keep running this + // specified container image. + // If not specified, the latest version compatible with the running CodeFlare operator is used. + // A CodeFlare operator upgrade may upgrade the InstaScale controller to a newer container image. + // + // +optional + ControllerImage string `json:"controllerImage,omitempty"` } // InstaScaleStatus defines the observed state of InstaScale @@ -48,8 +59,8 @@ type InstaScaleStatus struct { Ready bool `json:"ready"` } -//+kubebuilder:object:root=true -//+kubebuilder:subresource:status +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status // InstaScale is the Schema for the instascales API // +operator-sdk:csv:customresourcedefinitions:displayName="InstaScale" @@ -61,7 +72,7 @@ type InstaScale struct { Status InstaScaleStatus `json:"status,omitempty"` } -//+kubebuilder:object:root=true +// +kubebuilder:object:root=true // InstaScaleList contains a list of InstaScale type InstaScaleList struct { diff --git a/config/crd/bases/codeflare.codeflare.dev_instascales.yaml b/config/crd/bases/codeflare.codeflare.dev_instascales.yaml index 07a21ec74..26a72b760 100644 --- a/config/crd/bases/codeflare.codeflare.dev_instascales.yaml +++ b/config/crd/bases/codeflare.codeflare.dev_instascales.yaml @@ -35,6 +35,17 @@ spec: spec: description: InstaScaleSpec defines the desired state of InstaScale properties: + controllerImage: + description: The container image for the InstaScale controller deployment. + If specified, the provided container image must be compatible with + the running CodeFlare operator. Using an incompatible, or unrelated + container image, will result in an undefined behavior. A CodeFlare + operator upgrade will not upgrade the InstaScale controller, that'll + keep running this specified container image. If not specified, the + latest version compatible with the running CodeFlare operator is + used. A CodeFlare operator upgrade may upgrade the InstaScale controller + to a newer container image. + type: string controllerResources: description: controllerResources determines the container resources for the InstaScale controller deployment diff --git a/config/internal/instascale/deployment.yaml.tmpl b/config/internal/instascale/deployment.yaml.tmpl index 13df3a0da..4c704c80c 100644 --- a/config/internal/instascale/deployment.yaml.tmpl +++ b/config/internal/instascale/deployment.yaml.tmpl @@ -18,7 +18,7 @@ spec: containers: - args: - "--configs-namespace={{.Namespace}}" - image: quay.io/project-codeflare/instascale-controller:v0.0.3 + image: {{.ControllerImage}} name: instascale resources: {{.ControllerResources}} serviceAccountName: instascale-{{.Name}}-sa diff --git a/controllers/defaults.go b/controllers/defaults.go new file mode 100644 index 000000000..a0524c7c3 --- /dev/null +++ b/controllers/defaults.go @@ -0,0 +1,9 @@ +package controllers + +// *********************** +// DO NOT EDIT THIS FILE +// *********************** + +const ( + InstaScaleImage = "quay.io/project-codeflare/instascale-controller:v0.0.3" +) diff --git a/controllers/instascale_controller.go b/controllers/instascale_controller.go index 4b52bd123..c9fabdf57 100644 --- a/controllers/instascale_controller.go +++ b/controllers/instascale_controller.go @@ -19,6 +19,7 @@ package controllers import ( "context" "fmt" + "path" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -27,6 +28,7 @@ import ( apierrs "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -36,6 +38,7 @@ import ( "github.com/go-logr/logr" mf "github.com/manifestival/manifestival" + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/v1alpha1" "github.com/project-codeflare/codeflare-operator/controllers/config" "github.com/project-codeflare/codeflare-operator/controllers/util" @@ -55,8 +58,7 @@ var instascaleClusterScopedTemplates = []string{ } func (r *InstaScaleReconciler) Apply(owner mf.Owner, params *InstaScaleParams, template string, fns ...mf.Transformer) error { - - tmplManifest, err := config.Manifest(r.Client, r.TemplatesPath+template, params, template, r.Log) + tmplManifest, err := config.Manifest(r.Client, path.Join(r.TemplatesPath, template), params, template, r.Log) if err != nil { return fmt.Errorf("error loading template yaml: %w", err) } @@ -77,17 +79,17 @@ func (r *InstaScaleReconciler) Apply(owner mf.Owner, params *InstaScaleParams, t // TODO: Review node permissions, instascale should only require read -//+kubebuilder:rbac:groups=codeflare.codeflare.dev,resources=instascales,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=codeflare.codeflare.dev,resources=instascales/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=codeflare.codeflare.dev,resources=instascales/finalizers,verbs=update -//+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=*,resources=deployments;services,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=core,resources=secrets;configmaps;nodes;services;serviceaccounts;persistentvolumes;persistentvolumeclaims,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=core,resources=persistentvolumes;persistentvolumeclaims,verbs=* -//+kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterroles;clusterrolebindings,verbs=get;list;watch;create;update;delete -//+kubebuilder:rbac:groups=machine.openshift.io,resources=*,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=mcad.ibm.com,resources=appwrappers;queuejobs;schedulingspecs,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=codeflare.codeflare.dev,resources=instascales,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=codeflare.codeflare.dev,resources=instascales/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=codeflare.codeflare.dev,resources=instascales/finalizers,verbs=update +// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=*,resources=deployments;services,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=secrets;configmaps;nodes;services;serviceaccounts;persistentvolumes;persistentvolumeclaims,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=persistentvolumes;persistentvolumeclaims,verbs=* +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterroles;clusterrolebindings,verbs=get;list;watch;create;update;delete +// +kubebuilder:rbac:groups=machine.openshift.io,resources=*,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=mcad.ibm.com,resources=appwrappers;queuejobs;schedulingspecs,verbs=get;list;watch;create;update;patch;delete // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -124,11 +126,7 @@ func (r *InstaScaleReconciler) Reconcile(ctx context.Context, req ctrl.Request) instascaleCustomResource.APIVersion, instascaleCustomResource.Kind = gvk.Version, gvk.Kind } - err = params.ExtractParams(instascaleCustomResource) - if err != nil { - log.Error(err, "Unable to parse InstaScale custom resource") - return ctrl.Result{}, err - } + params.ExtractParams(instascaleCustomResource) if instascaleCustomResource.ObjectMeta.DeletionTimestamp.IsZero() { if !controllerutil.ContainsFinalizer(instascaleCustomResource, finalizerName) { @@ -152,7 +150,6 @@ func (r *InstaScaleReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } - log.V(1).Info("ReconcileInstaScale called.") err = r.ReconcileInstaScale(instascaleCustomResource, params) if err != nil { return ctrl.Result{}, err @@ -162,7 +159,8 @@ func (r *InstaScaleReconciler) Reconcile(ctx context.Context, req ctrl.Request) if err != nil { return ctrl.Result{}, err } - err = r.Client.Status().Update(context.Background(), instascaleCustomResource) + + err = r.Client.Status().Update(ctx, instascaleCustomResource) if err != nil { return ctrl.Result{}, err } diff --git a/controllers/instascale_params.go b/controllers/instascale_params.go index 1febfb39f..eb6737c79 100644 --- a/controllers/instascale_params.go +++ b/controllers/instascale_params.go @@ -3,21 +3,25 @@ package controllers import ( "encoding/json" - mf "github.com/manifestival/manifestival" - instascalev1alpha1 "github.com/project-codeflare/codeflare-operator/api/v1alpha1" + "github.com/manifestival/manifestival" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + + instascalev1alpha1 "github.com/project-codeflare/codeflare-operator/api/v1alpha1" ) type InstaScaleParams struct { Name string Namespace string - Owner mf.Owner + Owner manifestival.Owner EnableMonitoring bool MaxScaleoutAllowed int UseMachinePools bool ControllerResources ControllerResources + ControllerImage string } + type ControllerResources struct { v1.ResourceRequirements } @@ -29,9 +33,14 @@ func (c *ControllerResources) String() string { } return string(raw) } -func (p *InstaScaleParams) ExtractParams(instascale *instascalev1alpha1.InstaScale) error { + +func (p *InstaScaleParams) ExtractParams(instascale *instascalev1alpha1.InstaScale) { p.Name = instascale.Name p.Namespace = instascale.Namespace + p.ControllerImage = instascale.Spec.ControllerImage + if p.ControllerImage == "" { + p.ControllerImage = InstaScaleImage + } p.Owner = instascale p.EnableMonitoring = instascale.Spec.EnableMonitoring p.MaxScaleoutAllowed = instascale.Spec.MaxScaleoutAllowed @@ -49,5 +58,4 @@ func (p *InstaScaleParams) ExtractParams(instascale *instascalev1alpha1.InstaSca } else { p.ControllerResources = ControllerResources{*instascale.Spec.ControllerResources} } - return nil } From 59e97a4e55c446475db98f26cc2d71fc44d6c7f7 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 7 Jun 2023 11:49:09 +0200 Subject: [PATCH 006/100] Remove generated OLM bundle Dockerfile --- .gitignore | 3 +++ Makefile | 2 +- bundle.Dockerfile | 20 -------------------- 3 files changed, 4 insertions(+), 21 deletions(-) delete mode 100644 bundle.Dockerfile diff --git a/.gitignore b/.gitignore index 3b7689839..898d07796 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,10 @@ *.dylib bin testbin/* + +# OLM generate bundle files bundle/* +bundle.Dockerfile # Test binary, build with `go test -c` *.test diff --git a/Makefile b/Makefile index a6e5e6bae..41ec09de1 100644 --- a/Makefile +++ b/Makefile @@ -230,7 +230,7 @@ bundle: defaults manifests kustomize install-operator-sdk ## Generate bundle man $(MAKE) validate-bundle .PHONY: bundle-build -bundle-build: ## Build the bundle image. +bundle-build: bundle ## Build the bundle image. podman build -f bundle.Dockerfile -t $(BUNDLE_IMG) . .PHONY: bundle-push diff --git a/bundle.Dockerfile b/bundle.Dockerfile deleted file mode 100644 index 4f72266e7..000000000 --- a/bundle.Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM scratch - -# Core bundle labels. -LABEL operators.operatorframework.io.bundle.mediatype.v1=registry+v1 -LABEL operators.operatorframework.io.bundle.manifests.v1=manifests/ -LABEL operators.operatorframework.io.bundle.metadata.v1=metadata/ -LABEL operators.operatorframework.io.bundle.package.v1=codeflare-operator -LABEL operators.operatorframework.io.bundle.channels.v1=alpha -LABEL operators.operatorframework.io.metrics.builder=operator-sdk-v1.27.0 -LABEL operators.operatorframework.io.metrics.mediatype.v1=metrics+v1 -LABEL operators.operatorframework.io.metrics.project_layout=go.kubebuilder.io/v3 - -# Labels for testing. -LABEL operators.operatorframework.io.test.mediatype.v1=scorecard+v1 -LABEL operators.operatorframework.io.test.config.v1=tests/scorecard/ - -# Copy files to locations specified by labels. -COPY bundle/manifests /manifests/ -COPY bundle/metadata /metadata/ -COPY bundle/tests/scorecard /tests/scorecard/ From 08fa9ed11ca4e1bb3da9a297a08e81849a45c754 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 1 Jun 2023 10:05:22 +0200 Subject: [PATCH 007/100] Use uniquely identifying labels selector for the operator deployment --- config/default/manager_auth_proxy_patch.yaml | 2 +- config/default/manager_config_patch.yaml | 2 +- config/manager/manager.yaml | 8 +++++--- config/prometheus/monitor.yaml | 5 +++-- config/rbac/auth_proxy_service.yaml | 5 +++-- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml index cec149a07..24d5b1194 100644 --- a/config/default/manager_auth_proxy_patch.yaml +++ b/config/default/manager_auth_proxy_patch.yaml @@ -3,7 +3,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: controller-manager + name: manager namespace: system spec: template: diff --git a/config/default/manager_config_patch.yaml b/config/default/manager_config_patch.yaml index 6c400155c..98456c9a4 100644 --- a/config/default/manager_config_patch.yaml +++ b/config/default/manager_config_patch.yaml @@ -1,7 +1,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: controller-manager + name: manager namespace: system spec: template: diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index fb590870c..bdddb38a1 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -8,21 +8,23 @@ metadata: apiVersion: apps/v1 kind: Deployment metadata: - name: controller-manager + name: manager namespace: system labels: control-plane: controller-manager spec: selector: matchLabels: - control-plane: controller-manager + app.kubernetes.io/name: codeflare-operator + app.kubernetes.io/part-of: codeflare replicas: 1 template: metadata: annotations: kubectl.kubernetes.io/default-container: manager labels: - control-plane: controller-manager + app.kubernetes.io/name: codeflare-operator + app.kubernetes.io/part-of: codeflare spec: securityContext: runAsNonRoot: true diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml index 6812d4d5e..af832182d 100644 --- a/config/prometheus/monitor.yaml +++ b/config/prometheus/monitor.yaml @@ -4,7 +4,7 @@ kind: ServiceMonitor metadata: labels: control-plane: controller-manager - name: controller-manager-metrics-monitor + name: manager-metrics namespace: system spec: endpoints: @@ -16,4 +16,5 @@ spec: insecureSkipVerify: true selector: matchLabels: - control-plane: controller-manager + app.kubernetes.io/name: codeflare-operator + app.kubernetes.io/part-of: codeflare diff --git a/config/rbac/auth_proxy_service.yaml b/config/rbac/auth_proxy_service.yaml index 71f179727..86ffc6415 100644 --- a/config/rbac/auth_proxy_service.yaml +++ b/config/rbac/auth_proxy_service.yaml @@ -3,7 +3,7 @@ kind: Service metadata: labels: control-plane: controller-manager - name: controller-manager-metrics-service + name: manager-metrics namespace: system spec: ports: @@ -12,4 +12,5 @@ spec: protocol: TCP targetPort: https selector: - control-plane: controller-manager + app.kubernetes.io/name: codeflare-operator + app.kubernetes.io/part-of: codeflare From a71303b7d9a2a59eb8310dbbaa351e2b43af256c Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 1 Jun 2023 10:21:14 +0200 Subject: [PATCH 008/100] Label resources following recommended k8s labels convention --- config/default/kustomization.yaml | 5 +++-- config/manager/manager.yaml | 4 ---- config/prometheus/monitor.yaml | 2 -- config/rbac/auth_proxy_service.yaml | 2 -- 4 files changed, 3 insertions(+), 10 deletions(-) diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index b3c7d2079..1bfe8c541 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -9,8 +9,9 @@ namespace: openshift-operators namePrefix: codeflare-operator- # Labels to add to all resources and selectors. -# commonLabels: -# someName: someValue +commonLabels: + app.kubernetes.io/name: codeflare-operator + app.kubernetes.io/part-of: codeflare bases: - ../crd diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index bdddb38a1..59b41783f 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -1,8 +1,6 @@ apiVersion: v1 kind: Namespace metadata: - labels: - control-plane: controller-manager name: system --- apiVersion: apps/v1 @@ -10,8 +8,6 @@ kind: Deployment metadata: name: manager namespace: system - labels: - control-plane: controller-manager spec: selector: matchLabels: diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml index af832182d..3bcf6c747 100644 --- a/config/prometheus/monitor.yaml +++ b/config/prometheus/monitor.yaml @@ -2,8 +2,6 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: - labels: - control-plane: controller-manager name: manager-metrics namespace: system spec: diff --git a/config/rbac/auth_proxy_service.yaml b/config/rbac/auth_proxy_service.yaml index 86ffc6415..250100420 100644 --- a/config/rbac/auth_proxy_service.yaml +++ b/config/rbac/auth_proxy_service.yaml @@ -1,8 +1,6 @@ apiVersion: v1 kind: Service metadata: - labels: - control-plane: controller-manager name: manager-metrics namespace: system spec: From bb1ce801850d3cb24e440ba252c53ed421dc15d7 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 7 Jun 2023 10:12:49 +0200 Subject: [PATCH 009/100] Add test for custom InstaScale controller image support --- controllers/instascale_controller_test.go | 17 +++++++++-- .../instascale_test_cases/case_3.yaml | 6 ++++ .../case_3/deployment.yaml | 30 +++++++++++++++++++ 3 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 controllers/testdata/instascale_test_cases/case_3.yaml create mode 100644 controllers/testdata/instascale_test_results/case_3/deployment.yaml diff --git a/controllers/instascale_controller_test.go b/controllers/instascale_controller_test.go index ed7420a88..7017e681f 100644 --- a/controllers/instascale_controller_test.go +++ b/controllers/instascale_controller_test.go @@ -3,22 +3,26 @@ package controllers import ( "context" - mfc "github.com/manifestival/controller-runtime-client" - mf "github.com/manifestival/manifestival" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + + mfc "github.com/manifestival/controller-runtime-client" + mf "github.com/manifestival/manifestival" + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/v1alpha1" ) const ( instascaleCRCase1 = "./testdata/instascale_test_cases/case_1.yaml" instascaleCRCase2 = "./testdata/instascale_test_cases/case_2.yaml" + instascaleCRCase3 = "./testdata/instascale_test_cases/case_3.yaml" instascaleConfigMap1 = "./testdata/instascale_test_results/case_1/configmap.yaml" instascaleServiceAccount1 = "./testdata/instascale_test_results/case_1/serviceaccount.yaml" instascaleClusterRole1 = "./testdata/instascale_test_results/case_1/clusterrole.yaml" instascaleClusterRoleBinding1 = "./testdata/instascale_test_results/case_1/clusterrolebinding.yaml" instascaleDeployment1 = "./testdata/instascale_test_results/case_1/deployment.yaml" instascaleDeployment2 = "./testdata/instascale_test_results/case_2/deployment.yaml" + instascaleDeployment3 = "./testdata/instascale_test_results/case_3/deployment.yaml" ) func deployInstaScale(ctx context.Context, path string, opts mf.Option) { @@ -44,6 +48,7 @@ var _ = Describe("The Instascale Controller", func() { compareClusterRoleBindings(instascaleClusterRoleBinding1, opts) }) }) + Context("In a namespace, InstaScale ControllerResources is given", func() { It("It should deploy InstaScale with the given ControllerResources", func() { @@ -51,4 +56,12 @@ var _ = Describe("The Instascale Controller", func() { compareDeployments(instascaleDeployment2, opts) }) }) + + Context("When an InstaScale resource with a custom image is given", func() { + + It("It should deploy InstaScale with the given controller image", func() { + deployInstaScale(ctx, instascaleCRCase3, opts) + compareDeployments(instascaleDeployment3, opts) + }) + }) }) diff --git a/controllers/testdata/instascale_test_cases/case_3.yaml b/controllers/testdata/instascale_test_cases/case_3.yaml new file mode 100644 index 000000000..a7d807f06 --- /dev/null +++ b/controllers/testdata/instascale_test_cases/case_3.yaml @@ -0,0 +1,6 @@ +apiVersion: codeflare.codeflare.dev/v1alpha1 +kind: InstaScale +metadata: + name: example-custom-image +spec: + controllerImage: quay.io/project-codeflare/instascale-controller:custom diff --git a/controllers/testdata/instascale_test_results/case_3/deployment.yaml b/controllers/testdata/instascale_test_results/case_3/deployment.yaml new file mode 100644 index 000000000..0e6081103 --- /dev/null +++ b/controllers/testdata/instascale_test_results/case_3/deployment.yaml @@ -0,0 +1,30 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: instascale-example-custom-image + namespace: codeflare-operator-system + labels: + app: instascale-example-custom-image +spec: + replicas: 1 + selector: + matchLabels: + app: instascale-example-custom-image + template: + metadata: + labels: + app: instascale-example-custom-image + spec: + containers: + - name: instascale + args: + - "--configs-namespace=default" + image: quay.io/project-codeflare/instascale-controller:custom + resources: + limits: + cpu: '2' + memory: 2G + requests: + cpu: '200m' + memory: 200M + serviceAccountName: instascale-example-custom-image-sa From baa7738d4110d9ffc144319b7a80a7be91f930b5 Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Thu, 8 Jun 2023 17:41:06 +0000 Subject: [PATCH 010/100] Update operator to 0.0.4 Signed-off-by: Anish Asthana --- Makefile | 2 +- README.md | 10 ++++++++++ config/crd/mcad/kustomization.yaml | 2 +- config/internal/mcad/deployment.yaml.tmpl | 2 +- controllers/defaults.go | 2 +- 5 files changed, 14 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 41ec09de1..315293a04 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ PREVIOUS_VERSION ?= 0.0.3 VERSION ?= 0.0.3-dev # INSTASCALE_VERSION defines the default version of the InstaScale controller -INSTASCALE_VERSION ?= v0.0.3 +INSTASCALE_VERSION ?= v0.0.4 # CHANNELS define the bundle channels used in the bundle. # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") diff --git a/README.md b/README.md index e7b46b0c1..19ca46cff 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,12 @@ # codeflare-operator Operator for installation and lifecycle management of CodeFlare distributed workload stack, starting with MCAD and InstaScale + +CodeFlare Stack Compatibility Matrix + +| Component | Version | +|------------------------------|---------| +| CodeFlare Operator | v0.0.4 | +| Multi-Cluster App Dispatcher | v1.31.0 | +| CodeFlare-SDK | v0.4.4 | +| InstaScale | v0.0.4 | +| KubeRay | v0.5.0 | diff --git a/config/crd/mcad/kustomization.yaml b/config/crd/mcad/kustomization.yaml index d87d2f570..1474b25fc 100644 --- a/config/crd/mcad/kustomization.yaml +++ b/config/crd/mcad/kustomization.yaml @@ -1,2 +1,2 @@ resources: - - github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=main + - github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=release-v1.31.0 diff --git a/config/internal/mcad/deployment.yaml.tmpl b/config/internal/mcad/deployment.yaml.tmpl index 13d118ce3..7f6ebfa8d 100644 --- a/config/internal/mcad/deployment.yaml.tmpl +++ b/config/internal/mcad/deployment.yaml.tmpl @@ -25,7 +25,7 @@ spec: envFrom: - configMapRef: name: mcad-{{.Name}}-config - image: 'quay.io/project-codeflare/mcad-controller:release-v1.29.57' + image: 'quay.io/project-codeflare/mcad-controller:release-v1.31.0' imagePullPolicy: Always ports: - name: https diff --git a/controllers/defaults.go b/controllers/defaults.go index a0524c7c3..3f3fe49f0 100644 --- a/controllers/defaults.go +++ b/controllers/defaults.go @@ -5,5 +5,5 @@ package controllers // *********************** const ( - InstaScaleImage = "quay.io/project-codeflare/instascale-controller:v0.0.3" + InstaScaleImage = "quay.io/project-codeflare/instascale-controller:v0.0.4" ) From b3bdf17448c24bd8fa34f7bc2754def9eedd923f Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Thu, 8 Jun 2023 17:43:24 +0000 Subject: [PATCH 011/100] Add astefanutti to OWNERS Signed-off-by: Anish Asthana --- OWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/OWNERS b/OWNERS index 6dc8d6b7a..587540418 100644 --- a/OWNERS +++ b/OWNERS @@ -1,5 +1,6 @@ approvers: - anishasthana + - astefanutti - jbusche - kpostoffice - maxusmusti From 094529ca2ef755f92f3852a28cc8bb1eb3eb35f9 Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Thu, 8 Jun 2023 18:22:08 +0000 Subject: [PATCH 012/100] Fix failing InstaScale tests Signed-off-by: Anish Asthana --- .../testdata/instascale_test_results/case_1/deployment.yaml | 2 +- .../testdata/instascale_test_results/case_2/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/controllers/testdata/instascale_test_results/case_1/deployment.yaml b/controllers/testdata/instascale_test_results/case_1/deployment.yaml index f94cca0d4..31c3896fb 100644 --- a/controllers/testdata/instascale_test_results/case_1/deployment.yaml +++ b/controllers/testdata/instascale_test_results/case_1/deployment.yaml @@ -19,7 +19,7 @@ spec: - name: instascale args: - "--configs-namespace=default" - image: quay.io/project-codeflare/instascale-controller:v0.0.3 + image: quay.io/project-codeflare/instascale-controller:v0.0.4 resources: limits: cpu: '2' diff --git a/controllers/testdata/instascale_test_results/case_2/deployment.yaml b/controllers/testdata/instascale_test_results/case_2/deployment.yaml index beb59328e..eb259ec86 100644 --- a/controllers/testdata/instascale_test_results/case_2/deployment.yaml +++ b/controllers/testdata/instascale_test_results/case_2/deployment.yaml @@ -19,7 +19,7 @@ spec: - name: instascale args: - "--configs-namespace=default" - image: quay.io/project-codeflare/instascale-controller:v0.0.3 + image: quay.io/project-codeflare/instascale-controller:v0.0.4 resources: limits: cpu: '1' From 79520818e35d549e4cb57b2c67107a6753eb8d80 Mon Sep 17 00:00:00 2001 From: Kevin Date: Wed, 31 May 2023 11:10:36 -0400 Subject: [PATCH 013/100] Add make directive for opening PR on OpenShift community operators Signed-off-by: Kevin --- Makefile | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/Makefile b/Makefile index 315293a04..0163f68b6 100644 --- a/Makefile +++ b/Makefile @@ -187,11 +187,13 @@ KUSTOMIZE ?= $(LOCALBIN)/kustomize CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen ENVTEST ?= $(LOCALBIN)/setup-envtest OPERATOR_SDK ?= $(LOCALBIN)/operator-sdk +GH_CLI ?= $(LOCALBIN)/gh ## Tool Versions KUSTOMIZE_VERSION ?= v4.5.4 CONTROLLER_TOOLS_VERSION ?= v0.9.2 OPERATOR_SDK_VERSION ?= v1.27.0 +GH_CLI_VERSION ?= 2.30.0 KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" .PHONY: kustomize @@ -199,6 +201,17 @@ kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. $(KUSTOMIZE): $(LOCALBIN) test -s $(LOCALBIN)/kustomize || { curl -s $(KUSTOMIZE_INSTALL_SCRIPT) | bash -s -- $(subst v,,$(KUSTOMIZE_VERSION)) $(LOCALBIN); } +GH_CLI_DL_URL := https://github.com/cli/cli/releases/download/v$(GH_CLI_VERSION) +GH_CLI_DL_FILENAME := gh_$(GH_CLI_VERSION)_$(shell go env GOOS)_$(shell go env GOARCH) +.PHONY: install-gh-cli +install-gh-cli: $(GH_CLI) +$(GH_CLI): $(LOCALBIN) + curl -L $(GH_CLI_DL_URL)/$(GH_CLI_DL_FILENAME).tar.gz --output $(GH_CLI_DL_FILENAME).tar.gz + tar -xvzf $(GH_CLI_DL_FILENAME).tar.gz + cp $(GH_CLI_DL_FILENAME)/bin/gh $(GH_CLI) + rm -rf $(GH_CLI_DL_FILENAME) + rm $(GH_CLI_DL_FILENAME).tar.gz + .PHONY: controller-gen controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. $(CONTROLLER_GEN): $(LOCALBIN) @@ -237,6 +250,15 @@ bundle-build: bundle ## Build the bundle image. bundle-push: ## Push the bundle image. $(MAKE) image-push IMG=$(BUNDLE_IMG) +.PHONY: openshift-community-operator-releases +openshift-community-operator-release: install-gh-cli bundle ## build bundle and create PR in OpenShift community operators repository + gh repo clone git@github.com:project-codeflare/community-operators-prod.git + cd community-operators-prod && git pull upstream main && git push origin main + cp -r bundle community-operators-prod/operators/codeflare-operator/$(VERSION) + cd community-operators-prod && git checkout -b codeflare-release-$(VERSION) && git add operators/codeflare-operator/$(VERSION)/* && git commit -m "add bundle manifests codeflare version $(VERSION)" && git push origin codeflare-release-$(VERSION) + gh pr create --repo redhat-openshift-ecosystem/community-operators-prod --title "CodeFlare $(VERSION)" --body "New release of codeflare operator" --head project-codeflare:codeflare-release-$(VERSION) --base main + rm -rf community-operators-prod + .PHONY: opm OPM = ./bin/opm opm: ## Download opm locally if necessary. From d3cd099f9a4ef7f38a1ec3e265b20df154124ae6 Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Fri, 9 Jun 2023 08:28:47 -0400 Subject: [PATCH 014/100] Update version in makefile to 0.0.4 Signed-off-by: Anish Asthana --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0163f68b6..64367fa3c 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ # then we can add a patch in the `PHONY: bundle` PREVIOUS_VERSION ?= 0.0.3 -VERSION ?= 0.0.3-dev +VERSION ?= 0.0.4 # INSTASCALE_VERSION defines the default version of the InstaScale controller INSTASCALE_VERSION ?= v0.0.4 From 95a99358ed3c60a0ef9a814a6bec67647be85a39 Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 9 Jun 2023 09:50:05 -0400 Subject: [PATCH 015/100] add a sign to commit so that it passes git checkrun Signed-off-by: Kevin --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 64367fa3c..ffbcf3905 100644 --- a/Makefile +++ b/Makefile @@ -255,7 +255,7 @@ openshift-community-operator-release: install-gh-cli bundle ## build bundle and gh repo clone git@github.com:project-codeflare/community-operators-prod.git cd community-operators-prod && git pull upstream main && git push origin main cp -r bundle community-operators-prod/operators/codeflare-operator/$(VERSION) - cd community-operators-prod && git checkout -b codeflare-release-$(VERSION) && git add operators/codeflare-operator/$(VERSION)/* && git commit -m "add bundle manifests codeflare version $(VERSION)" && git push origin codeflare-release-$(VERSION) + cd community-operators-prod && git checkout -b codeflare-release-$(VERSION) && git add operators/codeflare-operator/$(VERSION)/* && git commit -s -m "add bundle manifests codeflare version $(VERSION)" && git push origin codeflare-release-$(VERSION) gh pr create --repo redhat-openshift-ecosystem/community-operators-prod --title "CodeFlare $(VERSION)" --body "New release of codeflare operator" --head project-codeflare:codeflare-release-$(VERSION) --base main rm -rf community-operators-prod From d4c9ecd492920ad39d01c52f9fa57ce3f5ba9837 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Tue, 6 Jun 2023 09:59:37 +0200 Subject: [PATCH 016/100] Update GitHub actions to avoid using deprecated actions --- .github/workflows/precommit.yml | 4 ++-- .github/workflows/tag-and-build.yml | 2 +- .github/workflows/unit_tests.yml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml index 32079cc6c..494e0dd35 100644 --- a/.github/workflows/precommit.yml +++ b/.github/workflows/precommit.yml @@ -21,10 +21,10 @@ jobs: volumes: - /cache steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Activate cache - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: /cache key: ${{ runner.os }}-cache-${{ hashFiles('**/go.sum', '.pre-commit-config.yaml') }} diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 28653862d..335c666c5 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@v3 - name: Activate cache - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: /cache key: ${{ runner.os }}-cache-${{ hashFiles('**/go.sum', '.pre-commit-config.yaml') }} diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index f5389ab85..f28c5b2b1 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -22,10 +22,10 @@ jobs: volumes: - /cache steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Activate cache - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: /cache key: ${{ runner.os }}-cache-${{ hashFiles('**/go.sum', '.pre-commit-config.yaml') }} From 071f0415802178bee29a97d97915cd9132fea259 Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 9 Jun 2023 10:36:26 -0400 Subject: [PATCH 017/100] change the default versions to be clearly placeholders Signed-off-by: Kevin --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index ffbcf3905..215bdefb8 100644 --- a/Makefile +++ b/Makefile @@ -6,8 +6,8 @@ # best if we could detect this. If we cannot, we need to document it somewhere. # then we can add a patch in the `PHONY: bundle` -PREVIOUS_VERSION ?= 0.0.3 -VERSION ?= 0.0.4 +PREVIOUS_VERSION ?= 0.0.0-dev +VERSION ?= 0.0.0-dev # INSTASCALE_VERSION defines the default version of the InstaScale controller INSTASCALE_VERSION ?= v0.0.4 From c1b5fa277658c521322a5c2ef52320d66aa6f2a5 Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 9 Jun 2023 10:53:12 -0400 Subject: [PATCH 018/100] update contributing guidelines with build info Signed-off-by: Kevin --- CONTRIBUTING.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6e74b783d..ff3ff8e4e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,14 +31,21 @@ If changes are made to any Go code (like in the `controllers` dir for example), - This will check and build/compile the modified code For building and pushing a new version of the operator image: - - `make image-build -e IMG=` - - `make image-push -e IMG` + - `make image-build -e IMG_TAG_BASE= VERSION=` + - `make image-push -e IMG_TAG_BASE= VERSION=` For deploying onto a cluster: - First, either set `KUBECONFIG` or ensure you are logged into a cluster in your environment - `make install` - `make deploy -e IMG=` +For building and pushing a new version of the bundled operator image: + - `make bundle-build -e IMG_TAG_BASE= VERSION= PREVIOUS_VERSION=` + - `make bundle-push -e IMG_TAG_BASE= VERSION= PREVIOUS_VERSION=` + +To create a new openshift-community-operator-release: + - `make openshift-community-operator-release -e IMG_TAG_BASE= VERSION= PREVIOUS_VERSION=` + ## Testing The CodeFlare Operator currently has unit tests and pre-commit checks - To enable and view pre-commit checks: `pre-commit install` From cf16a0787bc05e1b8036eec7a7d2c05a5eaca0c3 Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 9 Jun 2023 11:12:10 -0400 Subject: [PATCH 019/100] remove patches which are generated during bundle build Signed-off-by: Kevin --- config/manager/kustomization.yaml | 4 ---- config/manifests/kustomization.yaml | 7 ------- 2 files changed, 11 deletions(-) diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 145fc38d9..538c69f29 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -10,7 +10,3 @@ configMapGenerator: name: manager-config apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -images: -- name: controller - newName: quay.io/project-codeflare/codeflare-operator - newTag: v0.0.2 diff --git a/config/manifests/kustomization.yaml b/config/manifests/kustomization.yaml index ad366f2ef..954c255a0 100644 --- a/config/manifests/kustomization.yaml +++ b/config/manifests/kustomization.yaml @@ -7,10 +7,3 @@ resources: - ../scorecard apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -patches: -- patch: '[{"op":"add", "path":"/metadata/annotations/containerImage", "value": "quay.io/project-codeflare/codeflare-operator:v0.0.1" }]' - target: - kind: ClusterServiceVersion -- patch: '[{"op":"add", "path":"/metadata/annotations/containerImage", "value": "quay.io/project-codeflare/codeflare-operator:v0.0.2" }]' - target: - kind: ClusterServiceVersion From 360e01f386053e980766d915f8f1f4e471a71fa3 Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 9 Jun 2023 11:13:36 -0400 Subject: [PATCH 020/100] restore config directory after building bundle Signed-off-by: Kevin --- CONTRIBUTING.md | 10 +++++----- Makefile | 2 ++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ff3ff8e4e..85d98b387 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,8 +31,8 @@ If changes are made to any Go code (like in the `controllers` dir for example), - This will check and build/compile the modified code For building and pushing a new version of the operator image: - - `make image-build -e IMG_TAG_BASE= VERSION=` - - `make image-push -e IMG_TAG_BASE= VERSION=` + - `make image-build -e IMAGE_TAG_BASE= VERSION=` + - `make image-push -e IMAGE_TAG_BASE= VERSION=` For deploying onto a cluster: - First, either set `KUBECONFIG` or ensure you are logged into a cluster in your environment @@ -40,11 +40,11 @@ For deploying onto a cluster: - `make deploy -e IMG=` For building and pushing a new version of the bundled operator image: - - `make bundle-build -e IMG_TAG_BASE= VERSION= PREVIOUS_VERSION=` - - `make bundle-push -e IMG_TAG_BASE= VERSION= PREVIOUS_VERSION=` + - `make bundle-build -e IMAGE_TAG_BASE= VERSION= PREVIOUS_VERSION=` + - `make bundle-push -e IMAGE_TAG_BASE= VERSION= PREVIOUS_VERSION=` To create a new openshift-community-operator-release: - - `make openshift-community-operator-release -e IMG_TAG_BASE= VERSION= PREVIOUS_VERSION=` + - `make openshift-community-operator-release -e IMAGE_TAG_BASE= VERSION= PREVIOUS_VERSION=` ## Testing The CodeFlare Operator currently has unit tests and pre-commit checks diff --git a/Makefile b/Makefile index 215bdefb8..d5ce65fdc 100644 --- a/Makefile +++ b/Makefile @@ -170,6 +170,7 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} $(KUSTOMIZE) build config/default | kubectl apply -f - + git restore config/* .PHONY: undeploy undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. @@ -241,6 +242,7 @@ bundle: defaults manifests kustomize install-operator-sdk ## Generate bundle man cd config/manifests && $(KUSTOMIZE) edit add patch --patch '[{"op":"add", "path":"/spec/replaces", "value": "codeflare-operator.v$(PREVIOUS_VERSION)" }]' --kind ClusterServiceVersion $(KUSTOMIZE) build config/manifests | $(OPERATOR_SDK) generate bundle $(BUNDLE_GEN_FLAGS) $(MAKE) validate-bundle + git restore config/* .PHONY: bundle-build bundle-build: bundle ## Build the bundle image. From 31806041e8a977fda1a4d300460a47237a5558c7 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Mon, 12 Jun 2023 15:59:12 +0200 Subject: [PATCH 021/100] Generate CodeFlare API client --- Makefile | 60 ++++ .../v1alpha1/doc.go} | 19 +- .../v1alpha1/instascale_types.go | 5 +- api/{ => codeflare}/v1alpha1/mcad_types.go | 13 +- api/codeflare/v1alpha1/register.go | 51 ++++ .../v1alpha1/zz_generated.deepcopy.go | 2 +- .../codeflare/v1alpha1/instascale.go | 219 +++++++++++++++ .../codeflare/v1alpha1/instascalespec.go | 79 ++++++ .../codeflare/v1alpha1/instascalestatus.go | 39 +++ .../codeflare/v1alpha1/mcad.go | 219 +++++++++++++++ .../codeflare/v1alpha1/mcadspec.go | 106 ++++++++ .../codeflare/v1alpha1/mcadstatus.go | 39 +++ .../applyconfiguration/internal/internal.go | 62 +++++ client/applyconfiguration/utils.go | 47 ++++ client/clientset/versioned/clientset.go | 120 ++++++++ .../versioned/fake/clientset_generated.go | 85 ++++++ client/clientset/versioned/fake/doc.go | 20 ++ client/clientset/versioned/fake/register.go | 56 ++++ client/clientset/versioned/scheme/doc.go | 20 ++ client/clientset/versioned/scheme/register.go | 56 ++++ .../codeflare/v1alpha1/codeflare_client.go | 112 ++++++++ .../versioned/typed/codeflare/v1alpha1/doc.go | 20 ++ .../typed/codeflare/v1alpha1/fake/doc.go | 20 ++ .../v1alpha1/fake/fake_codeflare_client.go | 44 +++ .../v1alpha1/fake/fake_instascale.go | 189 +++++++++++++ .../codeflare/v1alpha1/fake/fake_mcad.go | 189 +++++++++++++ .../codeflare/v1alpha1/generated_expansion.go | 23 ++ .../typed/codeflare/v1alpha1/instascale.go | 256 ++++++++++++++++++ .../typed/codeflare/v1alpha1/mcad.go | 256 ++++++++++++++++++ .../externalversions/codeflare/interface.go | 46 ++++ .../codeflare/v1alpha1/instascale.go | 90 ++++++ .../codeflare/v1alpha1/interface.go | 52 ++++ .../codeflare/v1alpha1/mcad.go | 90 ++++++ client/informer/externalversions/factory.go | 251 +++++++++++++++++ client/informer/externalversions/generic.go | 64 +++++ .../internalinterfaces/factory_interfaces.go | 40 +++ .../codeflare/v1alpha1/expansion_generated.go | 35 +++ .../listers/codeflare/v1alpha1/instascale.go | 99 +++++++ client/listers/codeflare/v1alpha1/mcad.go | 99 +++++++ controllers/instascale.go | 2 +- controllers/instascale_controller.go | 10 +- controllers/instascale_controller_test.go | 2 +- controllers/instascale_params.go | 2 +- controllers/mcad_controller.go | 64 ++--- controllers/mcad_controller_test.go | 8 +- controllers/mcad_params.go | 18 +- controllers/multi_cluster_app_dispatcher.go | 18 +- controllers/suite_test.go | 30 +- go.mod | 2 +- main.go | 9 +- 50 files changed, 3364 insertions(+), 93 deletions(-) rename api/{v1alpha1/groupversion_info.go => codeflare/v1alpha1/doc.go} (51%) rename api/{ => codeflare}/v1alpha1/instascale_types.go (97%) rename api/{ => codeflare}/v1alpha1/mcad_types.go (93%) create mode 100644 api/codeflare/v1alpha1/register.go rename api/{ => codeflare}/v1alpha1/zz_generated.deepcopy.go (99%) create mode 100644 client/applyconfiguration/codeflare/v1alpha1/instascale.go create mode 100644 client/applyconfiguration/codeflare/v1alpha1/instascalespec.go create mode 100644 client/applyconfiguration/codeflare/v1alpha1/instascalestatus.go create mode 100644 client/applyconfiguration/codeflare/v1alpha1/mcad.go create mode 100644 client/applyconfiguration/codeflare/v1alpha1/mcadspec.go create mode 100644 client/applyconfiguration/codeflare/v1alpha1/mcadstatus.go create mode 100644 client/applyconfiguration/internal/internal.go create mode 100644 client/applyconfiguration/utils.go create mode 100644 client/clientset/versioned/clientset.go create mode 100644 client/clientset/versioned/fake/clientset_generated.go create mode 100644 client/clientset/versioned/fake/doc.go create mode 100644 client/clientset/versioned/fake/register.go create mode 100644 client/clientset/versioned/scheme/doc.go create mode 100644 client/clientset/versioned/scheme/register.go create mode 100644 client/clientset/versioned/typed/codeflare/v1alpha1/codeflare_client.go create mode 100644 client/clientset/versioned/typed/codeflare/v1alpha1/doc.go create mode 100644 client/clientset/versioned/typed/codeflare/v1alpha1/fake/doc.go create mode 100644 client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_codeflare_client.go create mode 100644 client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_instascale.go create mode 100644 client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_mcad.go create mode 100644 client/clientset/versioned/typed/codeflare/v1alpha1/generated_expansion.go create mode 100644 client/clientset/versioned/typed/codeflare/v1alpha1/instascale.go create mode 100644 client/clientset/versioned/typed/codeflare/v1alpha1/mcad.go create mode 100644 client/informer/externalversions/codeflare/interface.go create mode 100644 client/informer/externalversions/codeflare/v1alpha1/instascale.go create mode 100644 client/informer/externalversions/codeflare/v1alpha1/interface.go create mode 100644 client/informer/externalversions/codeflare/v1alpha1/mcad.go create mode 100644 client/informer/externalversions/factory.go create mode 100644 client/informer/externalversions/generic.go create mode 100644 client/informer/externalversions/internalinterfaces/factory_interfaces.go create mode 100644 client/listers/codeflare/v1alpha1/expansion_generated.go create mode 100644 client/listers/codeflare/v1alpha1/instascale.go create mode 100644 client/listers/codeflare/v1alpha1/mcad.go diff --git a/Makefile b/Makefile index d5ce65fdc..2fd51060f 100644 --- a/Makefile +++ b/Makefile @@ -125,6 +125,38 @@ manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and Cust generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." +.PHONY: generate-client ## Generate client packages +generate-client: code-generator + rm -rf client + $(APPLYCONFIGURATION_GEN) \ + --input-dirs="github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" \ + --go-header-file="hack/boilerplate.go.txt" \ + --output-package="github.com/project-codeflare/codeflare-operator/client/applyconfiguration" \ + --trim-path-prefix "github.com/project-codeflare/codeflare-operator" + $(CLIENT_GEN) \ + --input="codeflare/v1alpha1" \ + --input-base="github.com/project-codeflare/codeflare-operator/api" \ + --apply-configuration-package="github.com/project-codeflare/codeflare-operator/client/applyconfiguration" \ + --go-header-file="hack/boilerplate.go.txt" \ + --clientset-name "versioned" \ + --output-package="github.com/project-codeflare/codeflare-operator/client/clientset" \ + --output-base="." \ + --trim-path-prefix "github.com/project-codeflare/codeflare-operator" + $(LISTER_GEN) \ + --input-dirs="github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" \ + --go-header-file="hack/boilerplate.go.txt" \ + --output-base="." \ + --output-package="github.com/project-codeflare/codeflare-operator/client/listers" \ + --trim-path-prefix "github.com/project-codeflare/codeflare-operator" + $(INFORMER_GEN) \ + --input-dirs="github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" \ + --versioned-clientset-package="github.com/project-codeflare/codeflare-operator/client/clientset/versioned" \ + --listers-package="github.com/project-codeflare/codeflare-operator/client/listers" \ + --go-header-file="hack/boilerplate.go.txt" \ + --output-base="." \ + --output-package="github.com/project-codeflare/codeflare-operator/client/informer" \ + --trim-path-prefix "github.com/project-codeflare/codeflare-operator" + .PHONY: fmt fmt: ## Run go fmt against code. go fmt ./... @@ -185,6 +217,10 @@ $(LOCALBIN): ## Tool Binaries KUSTOMIZE ?= $(LOCALBIN)/kustomize +APPLYCONFIGURATION_GEN ?= $(LOCALBIN)/applyconfiguration-gen +CLIENT_GEN ?= $(LOCALBIN)/client-gen +LISTER_GEN ?= $(LOCALBIN)/lister-gen +INFORMER_GEN ?= $(LOCALBIN)/informer-gen CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen ENVTEST ?= $(LOCALBIN)/setup-envtest OPERATOR_SDK ?= $(LOCALBIN)/operator-sdk @@ -192,6 +228,7 @@ GH_CLI ?= $(LOCALBIN)/gh ## Tool Versions KUSTOMIZE_VERSION ?= v4.5.4 +CODEGEN_VERSION ?= v0.27.2 CONTROLLER_TOOLS_VERSION ?= v0.9.2 OPERATOR_SDK_VERSION ?= v1.27.0 GH_CLI_VERSION ?= 2.30.0 @@ -213,6 +250,29 @@ $(GH_CLI): $(LOCALBIN) rm -rf $(GH_CLI_DL_FILENAME) rm $(GH_CLI_DL_FILENAME).tar.gz +.PHONY: code-generator +code-generator: $(APPLYCONFIGURATION_GEN) $(CLIENT_GEN) $(LISTER_GEN) $(INFORMER_GEN) + +.PHONY: applyconfiguration-gen +applyconfiguration-gen: $(APPLYCONFIGURATION_GEN) +$(APPLYCONFIGURATION_GEN): $(LOCALBIN) + test -s $(LOCALBIN)/applyconfiguration-gen || GOBIN=$(LOCALBIN) go install k8s.io/code-generator/cmd/applyconfiguration-gen@$(CODEGEN_VERSION) + +.PHONY: client-gen +client-gen: $(CLIENT_GEN) +$(CLIENT_GEN): $(LOCALBIN) + test -s $(LOCALBIN)/client-gen || GOBIN=$(LOCALBIN) go install k8s.io/code-generator/cmd/client-gen@$(CODEGEN_VERSION) + +.PHONY: lister-gen +lister-gen: $(LISTER_GEN) +$(LISTER_GEN): $(LOCALBIN) + test -s $(LOCALBIN)/lister-gen || GOBIN=$(LOCALBIN) go install k8s.io/code-generator/cmd/lister-gen@$(CODEGEN_VERSION) + +.PHONY: informer-gen +informer-gen: $(INFORMER_GEN) +$(INFORMER_GEN): $(LOCALBIN) + test -s $(LOCALBIN)/informer-gen || GOBIN=$(LOCALBIN) go install k8s.io/code-generator/cmd/informer-gen@$(CODEGEN_VERSION) + .PHONY: controller-gen controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. $(CONTROLLER_GEN): $(LOCALBIN) diff --git a/api/v1alpha1/groupversion_info.go b/api/codeflare/v1alpha1/doc.go similarity index 51% rename from api/v1alpha1/groupversion_info.go rename to api/codeflare/v1alpha1/doc.go index cce3fbc09..13be1de49 100644 --- a/api/v1alpha1/groupversion_info.go +++ b/api/codeflare/v1alpha1/doc.go @@ -14,23 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package v1alpha1 contains API Schema definitions for the codeflare v1alpha1 API group +// +k8s:deepcopy-gen=package,register // +kubebuilder:object:generate=true // +groupName=codeflare.codeflare.dev -package v1alpha1 - -import ( - "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/controller-runtime/pkg/scheme" -) - -var ( - // GroupVersion is group version used to register these objects - GroupVersion = schema.GroupVersion{Group: "codeflare.codeflare.dev", Version: "v1alpha1"} - // SchemeBuilder is used to add go types to the GroupVersionKind scheme - SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} - - // AddToScheme adds the types in this group-version to the given scheme. - AddToScheme = SchemeBuilder.AddToScheme -) +package v1alpha1 diff --git a/api/v1alpha1/instascale_types.go b/api/codeflare/v1alpha1/instascale_types.go similarity index 97% rename from api/v1alpha1/instascale_types.go rename to api/codeflare/v1alpha1/instascale_types.go index 063f9a0d4..7368cb1a9 100644 --- a/api/v1alpha1/instascale_types.go +++ b/api/codeflare/v1alpha1/instascale_types.go @@ -59,6 +59,7 @@ type InstaScaleStatus struct { Ready bool `json:"ready"` } +// +genclient // +kubebuilder:object:root=true // +kubebuilder:subresource:status @@ -80,7 +81,3 @@ type InstaScaleList struct { metav1.ListMeta `json:"metadata,omitempty"` Items []InstaScale `json:"items"` } - -func init() { - SchemeBuilder.Register(&InstaScale{}, &InstaScaleList{}) -} diff --git a/api/v1alpha1/mcad_types.go b/api/codeflare/v1alpha1/mcad_types.go similarity index 93% rename from api/v1alpha1/mcad_types.go rename to api/codeflare/v1alpha1/mcad_types.go index e109bcaac..217791a21 100644 --- a/api/v1alpha1/mcad_types.go +++ b/api/codeflare/v1alpha1/mcad_types.go @@ -48,7 +48,7 @@ type MCADSpec struct { // PodCreationTimeout determines timeout in milliseconds for pods to be created after dispatching job. // +kubebuilder:default=-1 PodCreationTimeout int `json:"podCreationTimeout,omitempty"` - //podCreationTimeout: //int (default blank) + // podCreationTimeout: //int (default blank) // ControllerResources defines the cpu and memory resource requirements for the MCAD Controller // +kubebuilder:default={} @@ -63,8 +63,9 @@ type MCADStatus struct { Ready bool `json:"ready"` } -//+kubebuilder:object:root=true -//+kubebuilder:subresource:status +// +genclient +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status // MCAD is the Schema for the mcads API type MCAD struct { @@ -75,7 +76,7 @@ type MCAD struct { Status MCADStatus `json:"status,omitempty"` } -//+kubebuilder:object:root=true +// +kubebuilder:object:root=true // MCADList contains a list of MCAD type MCADList struct { @@ -83,7 +84,3 @@ type MCADList struct { metav1.ListMeta `json:"metadata,omitempty"` Items []MCAD `json:"items"` } - -func init() { - SchemeBuilder.Register(&MCAD{}, &MCADList{}) -} diff --git a/api/codeflare/v1alpha1/register.go b/api/codeflare/v1alpha1/register.go new file mode 100644 index 000000000..12bd39978 --- /dev/null +++ b/api/codeflare/v1alpha1/register.go @@ -0,0 +1,51 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// Kind takes an unqualified kind and returns back a Group qualified GroupKind. +func Kind(kind string) schema.GroupKind { + return SchemeGroupVersion.WithKind(kind).GroupKind() +} + +// Resource takes an unqualified resource and returns a Group qualified GroupResource. +func Resource(resource string) schema.GroupResource { + return SchemeGroupVersion.WithResource(resource).GroupResource() +} + +var ( + SchemeGroupVersion = schema.GroupVersion{Group: "codeflare.codeflare.dev", Version: "v1alpha1"} + SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + AddToScheme = SchemeBuilder.AddToScheme +) + +// Adds the list of known types to Scheme. +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(SchemeGroupVersion, + &MCAD{}, + &MCADList{}, + &InstaScale{}, + &InstaScaleList{}, + ) + metav1.AddToGroupVersion(scheme, SchemeGroupVersion) + return nil +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/codeflare/v1alpha1/zz_generated.deepcopy.go similarity index 99% rename from api/v1alpha1/zz_generated.deepcopy.go rename to api/codeflare/v1alpha1/zz_generated.deepcopy.go index 8a7f57135..5f0dd24ab 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/codeflare/v1alpha1/zz_generated.deepcopy.go @@ -23,7 +23,7 @@ package v1alpha1 import ( "k8s.io/api/core/v1" - runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. diff --git a/client/applyconfiguration/codeflare/v1alpha1/instascale.go b/client/applyconfiguration/codeflare/v1alpha1/instascale.go new file mode 100644 index 000000000..3cb4c8c67 --- /dev/null +++ b/client/applyconfiguration/codeflare/v1alpha1/instascale.go @@ -0,0 +1,219 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + v1 "k8s.io/client-go/applyconfigurations/meta/v1" +) + +// InstaScaleApplyConfiguration represents an declarative configuration of the InstaScale type for use +// with apply. +type InstaScaleApplyConfiguration struct { + v1.TypeMetaApplyConfiguration `json:",inline"` + *v1.ObjectMetaApplyConfiguration `json:"metadata,omitempty"` + Spec *InstaScaleSpecApplyConfiguration `json:"spec,omitempty"` + Status *InstaScaleStatusApplyConfiguration `json:"status,omitempty"` +} + +// InstaScale constructs an declarative configuration of the InstaScale type for use with +// apply. +func InstaScale(name, namespace string) *InstaScaleApplyConfiguration { + b := &InstaScaleApplyConfiguration{} + b.WithName(name) + b.WithNamespace(namespace) + b.WithKind("InstaScale") + b.WithAPIVersion("codeflare.codeflare.dev/v1alpha1") + return b +} + +// WithKind sets the Kind field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Kind field is set to the value of the last call. +func (b *InstaScaleApplyConfiguration) WithKind(value string) *InstaScaleApplyConfiguration { + b.Kind = &value + return b +} + +// WithAPIVersion sets the APIVersion field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the APIVersion field is set to the value of the last call. +func (b *InstaScaleApplyConfiguration) WithAPIVersion(value string) *InstaScaleApplyConfiguration { + b.APIVersion = &value + return b +} + +// WithName sets the Name field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Name field is set to the value of the last call. +func (b *InstaScaleApplyConfiguration) WithName(value string) *InstaScaleApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.Name = &value + return b +} + +// WithGenerateName sets the GenerateName field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the GenerateName field is set to the value of the last call. +func (b *InstaScaleApplyConfiguration) WithGenerateName(value string) *InstaScaleApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.GenerateName = &value + return b +} + +// WithNamespace sets the Namespace field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Namespace field is set to the value of the last call. +func (b *InstaScaleApplyConfiguration) WithNamespace(value string) *InstaScaleApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.Namespace = &value + return b +} + +// WithUID sets the UID field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the UID field is set to the value of the last call. +func (b *InstaScaleApplyConfiguration) WithUID(value types.UID) *InstaScaleApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.UID = &value + return b +} + +// WithResourceVersion sets the ResourceVersion field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the ResourceVersion field is set to the value of the last call. +func (b *InstaScaleApplyConfiguration) WithResourceVersion(value string) *InstaScaleApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ResourceVersion = &value + return b +} + +// WithGeneration sets the Generation field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Generation field is set to the value of the last call. +func (b *InstaScaleApplyConfiguration) WithGeneration(value int64) *InstaScaleApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.Generation = &value + return b +} + +// WithCreationTimestamp sets the CreationTimestamp field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the CreationTimestamp field is set to the value of the last call. +func (b *InstaScaleApplyConfiguration) WithCreationTimestamp(value metav1.Time) *InstaScaleApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.CreationTimestamp = &value + return b +} + +// WithDeletionTimestamp sets the DeletionTimestamp field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the DeletionTimestamp field is set to the value of the last call. +func (b *InstaScaleApplyConfiguration) WithDeletionTimestamp(value metav1.Time) *InstaScaleApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.DeletionTimestamp = &value + return b +} + +// WithDeletionGracePeriodSeconds sets the DeletionGracePeriodSeconds field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the DeletionGracePeriodSeconds field is set to the value of the last call. +func (b *InstaScaleApplyConfiguration) WithDeletionGracePeriodSeconds(value int64) *InstaScaleApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.DeletionGracePeriodSeconds = &value + return b +} + +// WithLabels puts the entries into the Labels field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, the entries provided by each call will be put on the Labels field, +// overwriting an existing map entries in Labels field with the same key. +func (b *InstaScaleApplyConfiguration) WithLabels(entries map[string]string) *InstaScaleApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + if b.Labels == nil && len(entries) > 0 { + b.Labels = make(map[string]string, len(entries)) + } + for k, v := range entries { + b.Labels[k] = v + } + return b +} + +// WithAnnotations puts the entries into the Annotations field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, the entries provided by each call will be put on the Annotations field, +// overwriting an existing map entries in Annotations field with the same key. +func (b *InstaScaleApplyConfiguration) WithAnnotations(entries map[string]string) *InstaScaleApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + if b.Annotations == nil && len(entries) > 0 { + b.Annotations = make(map[string]string, len(entries)) + } + for k, v := range entries { + b.Annotations[k] = v + } + return b +} + +// WithOwnerReferences adds the given value to the OwnerReferences field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the OwnerReferences field. +func (b *InstaScaleApplyConfiguration) WithOwnerReferences(values ...*v1.OwnerReferenceApplyConfiguration) *InstaScaleApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + for i := range values { + if values[i] == nil { + panic("nil value passed to WithOwnerReferences") + } + b.OwnerReferences = append(b.OwnerReferences, *values[i]) + } + return b +} + +// WithFinalizers adds the given value to the Finalizers field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the Finalizers field. +func (b *InstaScaleApplyConfiguration) WithFinalizers(values ...string) *InstaScaleApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + for i := range values { + b.Finalizers = append(b.Finalizers, values[i]) + } + return b +} + +func (b *InstaScaleApplyConfiguration) ensureObjectMetaApplyConfigurationExists() { + if b.ObjectMetaApplyConfiguration == nil { + b.ObjectMetaApplyConfiguration = &v1.ObjectMetaApplyConfiguration{} + } +} + +// WithSpec sets the Spec field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Spec field is set to the value of the last call. +func (b *InstaScaleApplyConfiguration) WithSpec(value *InstaScaleSpecApplyConfiguration) *InstaScaleApplyConfiguration { + b.Spec = value + return b +} + +// WithStatus sets the Status field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Status field is set to the value of the last call. +func (b *InstaScaleApplyConfiguration) WithStatus(value *InstaScaleStatusApplyConfiguration) *InstaScaleApplyConfiguration { + b.Status = value + return b +} diff --git a/client/applyconfiguration/codeflare/v1alpha1/instascalespec.go b/client/applyconfiguration/codeflare/v1alpha1/instascalespec.go new file mode 100644 index 000000000..e6ed5abd7 --- /dev/null +++ b/client/applyconfiguration/codeflare/v1alpha1/instascalespec.go @@ -0,0 +1,79 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + v1 "k8s.io/api/core/v1" +) + +// InstaScaleSpecApplyConfiguration represents an declarative configuration of the InstaScaleSpec type for use +// with apply. +type InstaScaleSpecApplyConfiguration struct { + EnableMonitoring *bool `json:"enableMonitoring,omitempty"` + MaxScaleoutAllowed *int `json:"maxScaleoutAllowed,omitempty"` + UseMachinePools *bool `json:"useMachinePools,omitempty"` + ControllerResources *v1.ResourceRequirements `json:"controllerResources,omitempty"` + ControllerImage *string `json:"controllerImage,omitempty"` +} + +// InstaScaleSpecApplyConfiguration constructs an declarative configuration of the InstaScaleSpec type for use with +// apply. +func InstaScaleSpec() *InstaScaleSpecApplyConfiguration { + return &InstaScaleSpecApplyConfiguration{} +} + +// WithEnableMonitoring sets the EnableMonitoring field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the EnableMonitoring field is set to the value of the last call. +func (b *InstaScaleSpecApplyConfiguration) WithEnableMonitoring(value bool) *InstaScaleSpecApplyConfiguration { + b.EnableMonitoring = &value + return b +} + +// WithMaxScaleoutAllowed sets the MaxScaleoutAllowed field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the MaxScaleoutAllowed field is set to the value of the last call. +func (b *InstaScaleSpecApplyConfiguration) WithMaxScaleoutAllowed(value int) *InstaScaleSpecApplyConfiguration { + b.MaxScaleoutAllowed = &value + return b +} + +// WithUseMachinePools sets the UseMachinePools field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the UseMachinePools field is set to the value of the last call. +func (b *InstaScaleSpecApplyConfiguration) WithUseMachinePools(value bool) *InstaScaleSpecApplyConfiguration { + b.UseMachinePools = &value + return b +} + +// WithControllerResources sets the ControllerResources field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the ControllerResources field is set to the value of the last call. +func (b *InstaScaleSpecApplyConfiguration) WithControllerResources(value v1.ResourceRequirements) *InstaScaleSpecApplyConfiguration { + b.ControllerResources = &value + return b +} + +// WithControllerImage sets the ControllerImage field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the ControllerImage field is set to the value of the last call. +func (b *InstaScaleSpecApplyConfiguration) WithControllerImage(value string) *InstaScaleSpecApplyConfiguration { + b.ControllerImage = &value + return b +} diff --git a/client/applyconfiguration/codeflare/v1alpha1/instascalestatus.go b/client/applyconfiguration/codeflare/v1alpha1/instascalestatus.go new file mode 100644 index 000000000..9643218f1 --- /dev/null +++ b/client/applyconfiguration/codeflare/v1alpha1/instascalestatus.go @@ -0,0 +1,39 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1alpha1 + +// InstaScaleStatusApplyConfiguration represents an declarative configuration of the InstaScaleStatus type for use +// with apply. +type InstaScaleStatusApplyConfiguration struct { + Ready *bool `json:"ready,omitempty"` +} + +// InstaScaleStatusApplyConfiguration constructs an declarative configuration of the InstaScaleStatus type for use with +// apply. +func InstaScaleStatus() *InstaScaleStatusApplyConfiguration { + return &InstaScaleStatusApplyConfiguration{} +} + +// WithReady sets the Ready field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Ready field is set to the value of the last call. +func (b *InstaScaleStatusApplyConfiguration) WithReady(value bool) *InstaScaleStatusApplyConfiguration { + b.Ready = &value + return b +} diff --git a/client/applyconfiguration/codeflare/v1alpha1/mcad.go b/client/applyconfiguration/codeflare/v1alpha1/mcad.go new file mode 100644 index 000000000..0b7a441b1 --- /dev/null +++ b/client/applyconfiguration/codeflare/v1alpha1/mcad.go @@ -0,0 +1,219 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + v1 "k8s.io/client-go/applyconfigurations/meta/v1" +) + +// MCADApplyConfiguration represents an declarative configuration of the MCAD type for use +// with apply. +type MCADApplyConfiguration struct { + v1.TypeMetaApplyConfiguration `json:",inline"` + *v1.ObjectMetaApplyConfiguration `json:"metadata,omitempty"` + Spec *MCADSpecApplyConfiguration `json:"spec,omitempty"` + Status *MCADStatusApplyConfiguration `json:"status,omitempty"` +} + +// MCAD constructs an declarative configuration of the MCAD type for use with +// apply. +func MCAD(name, namespace string) *MCADApplyConfiguration { + b := &MCADApplyConfiguration{} + b.WithName(name) + b.WithNamespace(namespace) + b.WithKind("MCAD") + b.WithAPIVersion("codeflare.codeflare.dev/v1alpha1") + return b +} + +// WithKind sets the Kind field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Kind field is set to the value of the last call. +func (b *MCADApplyConfiguration) WithKind(value string) *MCADApplyConfiguration { + b.Kind = &value + return b +} + +// WithAPIVersion sets the APIVersion field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the APIVersion field is set to the value of the last call. +func (b *MCADApplyConfiguration) WithAPIVersion(value string) *MCADApplyConfiguration { + b.APIVersion = &value + return b +} + +// WithName sets the Name field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Name field is set to the value of the last call. +func (b *MCADApplyConfiguration) WithName(value string) *MCADApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.Name = &value + return b +} + +// WithGenerateName sets the GenerateName field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the GenerateName field is set to the value of the last call. +func (b *MCADApplyConfiguration) WithGenerateName(value string) *MCADApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.GenerateName = &value + return b +} + +// WithNamespace sets the Namespace field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Namespace field is set to the value of the last call. +func (b *MCADApplyConfiguration) WithNamespace(value string) *MCADApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.Namespace = &value + return b +} + +// WithUID sets the UID field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the UID field is set to the value of the last call. +func (b *MCADApplyConfiguration) WithUID(value types.UID) *MCADApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.UID = &value + return b +} + +// WithResourceVersion sets the ResourceVersion field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the ResourceVersion field is set to the value of the last call. +func (b *MCADApplyConfiguration) WithResourceVersion(value string) *MCADApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ResourceVersion = &value + return b +} + +// WithGeneration sets the Generation field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Generation field is set to the value of the last call. +func (b *MCADApplyConfiguration) WithGeneration(value int64) *MCADApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.Generation = &value + return b +} + +// WithCreationTimestamp sets the CreationTimestamp field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the CreationTimestamp field is set to the value of the last call. +func (b *MCADApplyConfiguration) WithCreationTimestamp(value metav1.Time) *MCADApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.CreationTimestamp = &value + return b +} + +// WithDeletionTimestamp sets the DeletionTimestamp field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the DeletionTimestamp field is set to the value of the last call. +func (b *MCADApplyConfiguration) WithDeletionTimestamp(value metav1.Time) *MCADApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.DeletionTimestamp = &value + return b +} + +// WithDeletionGracePeriodSeconds sets the DeletionGracePeriodSeconds field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the DeletionGracePeriodSeconds field is set to the value of the last call. +func (b *MCADApplyConfiguration) WithDeletionGracePeriodSeconds(value int64) *MCADApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.DeletionGracePeriodSeconds = &value + return b +} + +// WithLabels puts the entries into the Labels field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, the entries provided by each call will be put on the Labels field, +// overwriting an existing map entries in Labels field with the same key. +func (b *MCADApplyConfiguration) WithLabels(entries map[string]string) *MCADApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + if b.Labels == nil && len(entries) > 0 { + b.Labels = make(map[string]string, len(entries)) + } + for k, v := range entries { + b.Labels[k] = v + } + return b +} + +// WithAnnotations puts the entries into the Annotations field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, the entries provided by each call will be put on the Annotations field, +// overwriting an existing map entries in Annotations field with the same key. +func (b *MCADApplyConfiguration) WithAnnotations(entries map[string]string) *MCADApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + if b.Annotations == nil && len(entries) > 0 { + b.Annotations = make(map[string]string, len(entries)) + } + for k, v := range entries { + b.Annotations[k] = v + } + return b +} + +// WithOwnerReferences adds the given value to the OwnerReferences field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the OwnerReferences field. +func (b *MCADApplyConfiguration) WithOwnerReferences(values ...*v1.OwnerReferenceApplyConfiguration) *MCADApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + for i := range values { + if values[i] == nil { + panic("nil value passed to WithOwnerReferences") + } + b.OwnerReferences = append(b.OwnerReferences, *values[i]) + } + return b +} + +// WithFinalizers adds the given value to the Finalizers field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the Finalizers field. +func (b *MCADApplyConfiguration) WithFinalizers(values ...string) *MCADApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + for i := range values { + b.Finalizers = append(b.Finalizers, values[i]) + } + return b +} + +func (b *MCADApplyConfiguration) ensureObjectMetaApplyConfigurationExists() { + if b.ObjectMetaApplyConfiguration == nil { + b.ObjectMetaApplyConfiguration = &v1.ObjectMetaApplyConfiguration{} + } +} + +// WithSpec sets the Spec field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Spec field is set to the value of the last call. +func (b *MCADApplyConfiguration) WithSpec(value *MCADSpecApplyConfiguration) *MCADApplyConfiguration { + b.Spec = value + return b +} + +// WithStatus sets the Status field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Status field is set to the value of the last call. +func (b *MCADApplyConfiguration) WithStatus(value *MCADStatusApplyConfiguration) *MCADApplyConfiguration { + b.Status = value + return b +} diff --git a/client/applyconfiguration/codeflare/v1alpha1/mcadspec.go b/client/applyconfiguration/codeflare/v1alpha1/mcadspec.go new file mode 100644 index 000000000..c7cb170a5 --- /dev/null +++ b/client/applyconfiguration/codeflare/v1alpha1/mcadspec.go @@ -0,0 +1,106 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + v1 "k8s.io/api/core/v1" +) + +// MCADSpecApplyConfiguration represents an declarative configuration of the MCADSpec type for use +// with apply. +type MCADSpecApplyConfiguration struct { + EnableMonitoring *bool `json:"enableMonitoring,omitempty"` + MultiCluster *bool `json:"multiCluster,omitempty"` + DispatcherMode *bool `json:"dispatcherMode,omitempty"` + PreemptionEnabled *bool `json:"preemptionEnabled,omitempty"` + AgentConfigs *string `json:"agentConfigs,omitempty"` + QuotaRestURL *string `json:"quotaRestURL,omitempty"` + PodCreationTimeout *int `json:"podCreationTimeout,omitempty"` + ControllerResources *v1.ResourceRequirements `json:"controllerResources,omitempty"` +} + +// MCADSpecApplyConfiguration constructs an declarative configuration of the MCADSpec type for use with +// apply. +func MCADSpec() *MCADSpecApplyConfiguration { + return &MCADSpecApplyConfiguration{} +} + +// WithEnableMonitoring sets the EnableMonitoring field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the EnableMonitoring field is set to the value of the last call. +func (b *MCADSpecApplyConfiguration) WithEnableMonitoring(value bool) *MCADSpecApplyConfiguration { + b.EnableMonitoring = &value + return b +} + +// WithMultiCluster sets the MultiCluster field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the MultiCluster field is set to the value of the last call. +func (b *MCADSpecApplyConfiguration) WithMultiCluster(value bool) *MCADSpecApplyConfiguration { + b.MultiCluster = &value + return b +} + +// WithDispatcherMode sets the DispatcherMode field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the DispatcherMode field is set to the value of the last call. +func (b *MCADSpecApplyConfiguration) WithDispatcherMode(value bool) *MCADSpecApplyConfiguration { + b.DispatcherMode = &value + return b +} + +// WithPreemptionEnabled sets the PreemptionEnabled field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the PreemptionEnabled field is set to the value of the last call. +func (b *MCADSpecApplyConfiguration) WithPreemptionEnabled(value bool) *MCADSpecApplyConfiguration { + b.PreemptionEnabled = &value + return b +} + +// WithAgentConfigs sets the AgentConfigs field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the AgentConfigs field is set to the value of the last call. +func (b *MCADSpecApplyConfiguration) WithAgentConfigs(value string) *MCADSpecApplyConfiguration { + b.AgentConfigs = &value + return b +} + +// WithQuotaRestURL sets the QuotaRestURL field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the QuotaRestURL field is set to the value of the last call. +func (b *MCADSpecApplyConfiguration) WithQuotaRestURL(value string) *MCADSpecApplyConfiguration { + b.QuotaRestURL = &value + return b +} + +// WithPodCreationTimeout sets the PodCreationTimeout field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the PodCreationTimeout field is set to the value of the last call. +func (b *MCADSpecApplyConfiguration) WithPodCreationTimeout(value int) *MCADSpecApplyConfiguration { + b.PodCreationTimeout = &value + return b +} + +// WithControllerResources sets the ControllerResources field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the ControllerResources field is set to the value of the last call. +func (b *MCADSpecApplyConfiguration) WithControllerResources(value v1.ResourceRequirements) *MCADSpecApplyConfiguration { + b.ControllerResources = &value + return b +} diff --git a/client/applyconfiguration/codeflare/v1alpha1/mcadstatus.go b/client/applyconfiguration/codeflare/v1alpha1/mcadstatus.go new file mode 100644 index 000000000..0b93b070a --- /dev/null +++ b/client/applyconfiguration/codeflare/v1alpha1/mcadstatus.go @@ -0,0 +1,39 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1alpha1 + +// MCADStatusApplyConfiguration represents an declarative configuration of the MCADStatus type for use +// with apply. +type MCADStatusApplyConfiguration struct { + Ready *bool `json:"ready,omitempty"` +} + +// MCADStatusApplyConfiguration constructs an declarative configuration of the MCADStatus type for use with +// apply. +func MCADStatus() *MCADStatusApplyConfiguration { + return &MCADStatusApplyConfiguration{} +} + +// WithReady sets the Ready field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Ready field is set to the value of the last call. +func (b *MCADStatusApplyConfiguration) WithReady(value bool) *MCADStatusApplyConfiguration { + b.Ready = &value + return b +} diff --git a/client/applyconfiguration/internal/internal.go b/client/applyconfiguration/internal/internal.go new file mode 100644 index 000000000..e3bfb7d5b --- /dev/null +++ b/client/applyconfiguration/internal/internal.go @@ -0,0 +1,62 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package internal + +import ( + "fmt" + "sync" + + typed "sigs.k8s.io/structured-merge-diff/v4/typed" +) + +func Parser() *typed.Parser { + parserOnce.Do(func() { + var err error + parser, err = typed.NewParser(schemaYAML) + if err != nil { + panic(fmt.Sprintf("Failed to parse schema: %v", err)) + } + }) + return parser +} + +var parserOnce sync.Once +var parser *typed.Parser +var schemaYAML = typed.YAMLObject(`types: +- name: __untyped_atomic_ + scalar: untyped + list: + elementType: + namedType: __untyped_atomic_ + elementRelationship: atomic + map: + elementType: + namedType: __untyped_atomic_ + elementRelationship: atomic +- name: __untyped_deduced_ + scalar: untyped + list: + elementType: + namedType: __untyped_atomic_ + elementRelationship: atomic + map: + elementType: + namedType: __untyped_deduced_ + elementRelationship: separable +`) diff --git a/client/applyconfiguration/utils.go b/client/applyconfiguration/utils.go new file mode 100644 index 000000000..fd87ac36b --- /dev/null +++ b/client/applyconfiguration/utils.go @@ -0,0 +1,47 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package applyconfiguration + +import ( + v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/applyconfiguration/codeflare/v1alpha1" + schema "k8s.io/apimachinery/pkg/runtime/schema" +) + +// ForKind returns an apply configuration type for the given GroupVersionKind, or nil if no +// apply configuration type exists for the given GroupVersionKind. +func ForKind(kind schema.GroupVersionKind) interface{} { + switch kind { + // Group=codeflare.codeflare.dev, Version=v1alpha1 + case v1alpha1.SchemeGroupVersion.WithKind("InstaScale"): + return &codeflarev1alpha1.InstaScaleApplyConfiguration{} + case v1alpha1.SchemeGroupVersion.WithKind("InstaScaleSpec"): + return &codeflarev1alpha1.InstaScaleSpecApplyConfiguration{} + case v1alpha1.SchemeGroupVersion.WithKind("InstaScaleStatus"): + return &codeflarev1alpha1.InstaScaleStatusApplyConfiguration{} + case v1alpha1.SchemeGroupVersion.WithKind("MCAD"): + return &codeflarev1alpha1.MCADApplyConfiguration{} + case v1alpha1.SchemeGroupVersion.WithKind("MCADSpec"): + return &codeflarev1alpha1.MCADSpecApplyConfiguration{} + case v1alpha1.SchemeGroupVersion.WithKind("MCADStatus"): + return &codeflarev1alpha1.MCADStatusApplyConfiguration{} + + } + return nil +} diff --git a/client/clientset/versioned/clientset.go b/client/clientset/versioned/clientset.go new file mode 100644 index 000000000..1d91a6641 --- /dev/null +++ b/client/clientset/versioned/clientset.go @@ -0,0 +1,120 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package versioned + +import ( + "fmt" + "net/http" + + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/typed/codeflare/v1alpha1" + discovery "k8s.io/client-go/discovery" + rest "k8s.io/client-go/rest" + flowcontrol "k8s.io/client-go/util/flowcontrol" +) + +type Interface interface { + Discovery() discovery.DiscoveryInterface + CodeflareV1alpha1() codeflarev1alpha1.CodeflareV1alpha1Interface +} + +// Clientset contains the clients for groups. +type Clientset struct { + *discovery.DiscoveryClient + codeflareV1alpha1 *codeflarev1alpha1.CodeflareV1alpha1Client +} + +// CodeflareV1alpha1 retrieves the CodeflareV1alpha1Client +func (c *Clientset) CodeflareV1alpha1() codeflarev1alpha1.CodeflareV1alpha1Interface { + return c.codeflareV1alpha1 +} + +// Discovery retrieves the DiscoveryClient +func (c *Clientset) Discovery() discovery.DiscoveryInterface { + if c == nil { + return nil + } + return c.DiscoveryClient +} + +// NewForConfig creates a new Clientset for the given config. +// If config's RateLimiter is not set and QPS and Burst are acceptable, +// NewForConfig will generate a rate-limiter in configShallowCopy. +// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), +// where httpClient was generated with rest.HTTPClientFor(c). +func NewForConfig(c *rest.Config) (*Clientset, error) { + configShallowCopy := *c + + if configShallowCopy.UserAgent == "" { + configShallowCopy.UserAgent = rest.DefaultKubernetesUserAgent() + } + + // share the transport between all clients + httpClient, err := rest.HTTPClientFor(&configShallowCopy) + if err != nil { + return nil, err + } + + return NewForConfigAndClient(&configShallowCopy, httpClient) +} + +// NewForConfigAndClient creates a new Clientset for the given config and http client. +// Note the http client provided takes precedence over the configured transport values. +// If config's RateLimiter is not set and QPS and Burst are acceptable, +// NewForConfigAndClient will generate a rate-limiter in configShallowCopy. +func NewForConfigAndClient(c *rest.Config, httpClient *http.Client) (*Clientset, error) { + configShallowCopy := *c + if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 { + if configShallowCopy.Burst <= 0 { + return nil, fmt.Errorf("burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0") + } + configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst) + } + + var cs Clientset + var err error + cs.codeflareV1alpha1, err = codeflarev1alpha1.NewForConfigAndClient(&configShallowCopy, httpClient) + if err != nil { + return nil, err + } + + cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfigAndClient(&configShallowCopy, httpClient) + if err != nil { + return nil, err + } + return &cs, nil +} + +// NewForConfigOrDie creates a new Clientset for the given config and +// panics if there is an error in the config. +func NewForConfigOrDie(c *rest.Config) *Clientset { + cs, err := NewForConfig(c) + if err != nil { + panic(err) + } + return cs +} + +// New creates a new Clientset for the given RESTClient. +func New(c rest.Interface) *Clientset { + var cs Clientset + cs.codeflareV1alpha1 = codeflarev1alpha1.New(c) + + cs.DiscoveryClient = discovery.NewDiscoveryClient(c) + return &cs +} diff --git a/client/clientset/versioned/fake/clientset_generated.go b/client/clientset/versioned/fake/clientset_generated.go new file mode 100644 index 000000000..fbbb80dc3 --- /dev/null +++ b/client/clientset/versioned/fake/clientset_generated.go @@ -0,0 +1,85 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + clientset "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/typed/codeflare/v1alpha1" + fakecodeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/typed/codeflare/v1alpha1/fake" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/discovery" + fakediscovery "k8s.io/client-go/discovery/fake" + "k8s.io/client-go/testing" +) + +// NewSimpleClientset returns a clientset that will respond with the provided objects. +// It's backed by a very simple object tracker that processes creates, updates and deletions as-is, +// without applying any validations and/or defaults. It shouldn't be considered a replacement +// for a real clientset and is mostly useful in simple unit tests. +func NewSimpleClientset(objects ...runtime.Object) *Clientset { + o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder()) + for _, obj := range objects { + if err := o.Add(obj); err != nil { + panic(err) + } + } + + cs := &Clientset{tracker: o} + cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake} + cs.AddReactor("*", "*", testing.ObjectReaction(o)) + cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) { + gvr := action.GetResource() + ns := action.GetNamespace() + watch, err := o.Watch(gvr, ns) + if err != nil { + return false, nil, err + } + return true, watch, nil + }) + + return cs +} + +// Clientset implements clientset.Interface. Meant to be embedded into a +// struct to get a default implementation. This makes faking out just the method +// you want to test easier. +type Clientset struct { + testing.Fake + discovery *fakediscovery.FakeDiscovery + tracker testing.ObjectTracker +} + +func (c *Clientset) Discovery() discovery.DiscoveryInterface { + return c.discovery +} + +func (c *Clientset) Tracker() testing.ObjectTracker { + return c.tracker +} + +var ( + _ clientset.Interface = &Clientset{} + _ testing.FakeClient = &Clientset{} +) + +// CodeflareV1alpha1 retrieves the CodeflareV1alpha1Client +func (c *Clientset) CodeflareV1alpha1() codeflarev1alpha1.CodeflareV1alpha1Interface { + return &fakecodeflarev1alpha1.FakeCodeflareV1alpha1{Fake: &c.Fake} +} diff --git a/client/clientset/versioned/fake/doc.go b/client/clientset/versioned/fake/doc.go new file mode 100644 index 000000000..86aee2a9e --- /dev/null +++ b/client/clientset/versioned/fake/doc.go @@ -0,0 +1,20 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated fake clientset. +package fake diff --git a/client/clientset/versioned/fake/register.go b/client/clientset/versioned/fake/register.go new file mode 100644 index 000000000..4ff1cb3a3 --- /dev/null +++ b/client/clientset/versioned/fake/register.go @@ -0,0 +1,56 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + serializer "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" +) + +var scheme = runtime.NewScheme() +var codecs = serializer.NewCodecFactory(scheme) + +var localSchemeBuilder = runtime.SchemeBuilder{ + codeflarev1alpha1.AddToScheme, +} + +// AddToScheme adds all types of this clientset into the given scheme. This allows composition +// of clientsets, like in: +// +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) +// +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// +// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types +// correctly. +var AddToScheme = localSchemeBuilder.AddToScheme + +func init() { + v1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"}) + utilruntime.Must(AddToScheme(scheme)) +} diff --git a/client/clientset/versioned/scheme/doc.go b/client/clientset/versioned/scheme/doc.go new file mode 100644 index 000000000..5f6297fd6 --- /dev/null +++ b/client/clientset/versioned/scheme/doc.go @@ -0,0 +1,20 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +// This package contains the scheme of the automatically generated clientset. +package scheme diff --git a/client/clientset/versioned/scheme/register.go b/client/clientset/versioned/scheme/register.go new file mode 100644 index 000000000..743173cea --- /dev/null +++ b/client/clientset/versioned/scheme/register.go @@ -0,0 +1,56 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package scheme + +import ( + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + serializer "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" +) + +var Scheme = runtime.NewScheme() +var Codecs = serializer.NewCodecFactory(Scheme) +var ParameterCodec = runtime.NewParameterCodec(Scheme) +var localSchemeBuilder = runtime.SchemeBuilder{ + codeflarev1alpha1.AddToScheme, +} + +// AddToScheme adds all types of this clientset into the given scheme. This allows composition +// of clientsets, like in: +// +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) +// +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// +// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types +// correctly. +var AddToScheme = localSchemeBuilder.AddToScheme + +func init() { + v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) + utilruntime.Must(AddToScheme(Scheme)) +} diff --git a/client/clientset/versioned/typed/codeflare/v1alpha1/codeflare_client.go b/client/clientset/versioned/typed/codeflare/v1alpha1/codeflare_client.go new file mode 100644 index 000000000..4b93fe6fe --- /dev/null +++ b/client/clientset/versioned/typed/codeflare/v1alpha1/codeflare_client.go @@ -0,0 +1,112 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "net/http" + + v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" + "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/scheme" + rest "k8s.io/client-go/rest" +) + +type CodeflareV1alpha1Interface interface { + RESTClient() rest.Interface + InstaScalesGetter + MCADsGetter +} + +// CodeflareV1alpha1Client is used to interact with features provided by the codeflare.codeflare.dev group. +type CodeflareV1alpha1Client struct { + restClient rest.Interface +} + +func (c *CodeflareV1alpha1Client) InstaScales(namespace string) InstaScaleInterface { + return newInstaScales(c, namespace) +} + +func (c *CodeflareV1alpha1Client) MCADs(namespace string) MCADInterface { + return newMCADs(c, namespace) +} + +// NewForConfig creates a new CodeflareV1alpha1Client for the given config. +// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), +// where httpClient was generated with rest.HTTPClientFor(c). +func NewForConfig(c *rest.Config) (*CodeflareV1alpha1Client, error) { + config := *c + if err := setConfigDefaults(&config); err != nil { + return nil, err + } + httpClient, err := rest.HTTPClientFor(&config) + if err != nil { + return nil, err + } + return NewForConfigAndClient(&config, httpClient) +} + +// NewForConfigAndClient creates a new CodeflareV1alpha1Client for the given config and http client. +// Note the http client provided takes precedence over the configured transport values. +func NewForConfigAndClient(c *rest.Config, h *http.Client) (*CodeflareV1alpha1Client, error) { + config := *c + if err := setConfigDefaults(&config); err != nil { + return nil, err + } + client, err := rest.RESTClientForConfigAndClient(&config, h) + if err != nil { + return nil, err + } + return &CodeflareV1alpha1Client{client}, nil +} + +// NewForConfigOrDie creates a new CodeflareV1alpha1Client for the given config and +// panics if there is an error in the config. +func NewForConfigOrDie(c *rest.Config) *CodeflareV1alpha1Client { + client, err := NewForConfig(c) + if err != nil { + panic(err) + } + return client +} + +// New creates a new CodeflareV1alpha1Client for the given RESTClient. +func New(c rest.Interface) *CodeflareV1alpha1Client { + return &CodeflareV1alpha1Client{c} +} + +func setConfigDefaults(config *rest.Config) error { + gv := v1alpha1.SchemeGroupVersion + config.GroupVersion = &gv + config.APIPath = "/apis" + config.NegotiatedSerializer = scheme.Codecs.WithoutConversion() + + if config.UserAgent == "" { + config.UserAgent = rest.DefaultKubernetesUserAgent() + } + + return nil +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *CodeflareV1alpha1Client) RESTClient() rest.Interface { + if c == nil { + return nil + } + return c.restClient +} diff --git a/client/clientset/versioned/typed/codeflare/v1alpha1/doc.go b/client/clientset/versioned/typed/codeflare/v1alpha1/doc.go new file mode 100644 index 000000000..cf7c15204 --- /dev/null +++ b/client/clientset/versioned/typed/codeflare/v1alpha1/doc.go @@ -0,0 +1,20 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated typed clients. +package v1alpha1 diff --git a/client/clientset/versioned/typed/codeflare/v1alpha1/fake/doc.go b/client/clientset/versioned/typed/codeflare/v1alpha1/fake/doc.go new file mode 100644 index 000000000..7c26ebe20 --- /dev/null +++ b/client/clientset/versioned/typed/codeflare/v1alpha1/fake/doc.go @@ -0,0 +1,20 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +// Package fake has the automatically generated clients. +package fake diff --git a/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_codeflare_client.go b/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_codeflare_client.go new file mode 100644 index 000000000..9e234aca7 --- /dev/null +++ b/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_codeflare_client.go @@ -0,0 +1,44 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1alpha1 "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/typed/codeflare/v1alpha1" + rest "k8s.io/client-go/rest" + testing "k8s.io/client-go/testing" +) + +type FakeCodeflareV1alpha1 struct { + *testing.Fake +} + +func (c *FakeCodeflareV1alpha1) InstaScales(namespace string) v1alpha1.InstaScaleInterface { + return &FakeInstaScales{c, namespace} +} + +func (c *FakeCodeflareV1alpha1) MCADs(namespace string) v1alpha1.MCADInterface { + return &FakeMCADs{c, namespace} +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *FakeCodeflareV1alpha1) RESTClient() rest.Interface { + var ret *rest.RESTClient + return ret +} diff --git a/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_instascale.go b/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_instascale.go new file mode 100644 index 000000000..8abbdc792 --- /dev/null +++ b/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_instascale.go @@ -0,0 +1,189 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + json "encoding/json" + "fmt" + + v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/applyconfiguration/codeflare/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeInstaScales implements InstaScaleInterface +type FakeInstaScales struct { + Fake *FakeCodeflareV1alpha1 + ns string +} + +var instascalesResource = v1alpha1.SchemeGroupVersion.WithResource("instascales") + +var instascalesKind = v1alpha1.SchemeGroupVersion.WithKind("InstaScale") + +// Get takes name of the instaScale, and returns the corresponding instaScale object, and an error if there is any. +func (c *FakeInstaScales) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.InstaScale, err error) { + obj, err := c.Fake. + Invokes(testing.NewGetAction(instascalesResource, c.ns, name), &v1alpha1.InstaScale{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.InstaScale), err +} + +// List takes label and field selectors, and returns the list of InstaScales that match those selectors. +func (c *FakeInstaScales) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.InstaScaleList, err error) { + obj, err := c.Fake. + Invokes(testing.NewListAction(instascalesResource, instascalesKind, c.ns, opts), &v1alpha1.InstaScaleList{}) + + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha1.InstaScaleList{ListMeta: obj.(*v1alpha1.InstaScaleList).ListMeta} + for _, item := range obj.(*v1alpha1.InstaScaleList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested instaScales. +func (c *FakeInstaScales) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchAction(instascalesResource, c.ns, opts)) + +} + +// Create takes the representation of a instaScale and creates it. Returns the server's representation of the instaScale, and an error, if there is any. +func (c *FakeInstaScales) Create(ctx context.Context, instaScale *v1alpha1.InstaScale, opts v1.CreateOptions) (result *v1alpha1.InstaScale, err error) { + obj, err := c.Fake. + Invokes(testing.NewCreateAction(instascalesResource, c.ns, instaScale), &v1alpha1.InstaScale{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.InstaScale), err +} + +// Update takes the representation of a instaScale and updates it. Returns the server's representation of the instaScale, and an error, if there is any. +func (c *FakeInstaScales) Update(ctx context.Context, instaScale *v1alpha1.InstaScale, opts v1.UpdateOptions) (result *v1alpha1.InstaScale, err error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateAction(instascalesResource, c.ns, instaScale), &v1alpha1.InstaScale{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.InstaScale), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeInstaScales) UpdateStatus(ctx context.Context, instaScale *v1alpha1.InstaScale, opts v1.UpdateOptions) (*v1alpha1.InstaScale, error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateSubresourceAction(instascalesResource, "status", c.ns, instaScale), &v1alpha1.InstaScale{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.InstaScale), err +} + +// Delete takes name of the instaScale and deletes it. Returns an error if one occurs. +func (c *FakeInstaScales) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteActionWithOptions(instascalesResource, c.ns, name, opts), &v1alpha1.InstaScale{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeInstaScales) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionAction(instascalesResource, c.ns, listOpts) + + _, err := c.Fake.Invokes(action, &v1alpha1.InstaScaleList{}) + return err +} + +// Patch applies the patch and returns the patched instaScale. +func (c *FakeInstaScales) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.InstaScale, err error) { + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceAction(instascalesResource, c.ns, name, pt, data, subresources...), &v1alpha1.InstaScale{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.InstaScale), err +} + +// Apply takes the given apply declarative configuration, applies it and returns the applied instaScale. +func (c *FakeInstaScales) Apply(ctx context.Context, instaScale *codeflarev1alpha1.InstaScaleApplyConfiguration, opts v1.ApplyOptions) (result *v1alpha1.InstaScale, err error) { + if instaScale == nil { + return nil, fmt.Errorf("instaScale provided to Apply must not be nil") + } + data, err := json.Marshal(instaScale) + if err != nil { + return nil, err + } + name := instaScale.Name + if name == nil { + return nil, fmt.Errorf("instaScale.Name must be provided to Apply") + } + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceAction(instascalesResource, c.ns, *name, types.ApplyPatchType, data), &v1alpha1.InstaScale{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.InstaScale), err +} + +// ApplyStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating ApplyStatus(). +func (c *FakeInstaScales) ApplyStatus(ctx context.Context, instaScale *codeflarev1alpha1.InstaScaleApplyConfiguration, opts v1.ApplyOptions) (result *v1alpha1.InstaScale, err error) { + if instaScale == nil { + return nil, fmt.Errorf("instaScale provided to Apply must not be nil") + } + data, err := json.Marshal(instaScale) + if err != nil { + return nil, err + } + name := instaScale.Name + if name == nil { + return nil, fmt.Errorf("instaScale.Name must be provided to Apply") + } + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceAction(instascalesResource, c.ns, *name, types.ApplyPatchType, data, "status"), &v1alpha1.InstaScale{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.InstaScale), err +} diff --git a/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_mcad.go b/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_mcad.go new file mode 100644 index 000000000..027f2817d --- /dev/null +++ b/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_mcad.go @@ -0,0 +1,189 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + json "encoding/json" + "fmt" + + v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/applyconfiguration/codeflare/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeMCADs implements MCADInterface +type FakeMCADs struct { + Fake *FakeCodeflareV1alpha1 + ns string +} + +var mcadsResource = v1alpha1.SchemeGroupVersion.WithResource("mcads") + +var mcadsKind = v1alpha1.SchemeGroupVersion.WithKind("MCAD") + +// Get takes name of the mCAD, and returns the corresponding mCAD object, and an error if there is any. +func (c *FakeMCADs) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.MCAD, err error) { + obj, err := c.Fake. + Invokes(testing.NewGetAction(mcadsResource, c.ns, name), &v1alpha1.MCAD{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.MCAD), err +} + +// List takes label and field selectors, and returns the list of MCADs that match those selectors. +func (c *FakeMCADs) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.MCADList, err error) { + obj, err := c.Fake. + Invokes(testing.NewListAction(mcadsResource, mcadsKind, c.ns, opts), &v1alpha1.MCADList{}) + + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha1.MCADList{ListMeta: obj.(*v1alpha1.MCADList).ListMeta} + for _, item := range obj.(*v1alpha1.MCADList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested mCADs. +func (c *FakeMCADs) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchAction(mcadsResource, c.ns, opts)) + +} + +// Create takes the representation of a mCAD and creates it. Returns the server's representation of the mCAD, and an error, if there is any. +func (c *FakeMCADs) Create(ctx context.Context, mCAD *v1alpha1.MCAD, opts v1.CreateOptions) (result *v1alpha1.MCAD, err error) { + obj, err := c.Fake. + Invokes(testing.NewCreateAction(mcadsResource, c.ns, mCAD), &v1alpha1.MCAD{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.MCAD), err +} + +// Update takes the representation of a mCAD and updates it. Returns the server's representation of the mCAD, and an error, if there is any. +func (c *FakeMCADs) Update(ctx context.Context, mCAD *v1alpha1.MCAD, opts v1.UpdateOptions) (result *v1alpha1.MCAD, err error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateAction(mcadsResource, c.ns, mCAD), &v1alpha1.MCAD{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.MCAD), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeMCADs) UpdateStatus(ctx context.Context, mCAD *v1alpha1.MCAD, opts v1.UpdateOptions) (*v1alpha1.MCAD, error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateSubresourceAction(mcadsResource, "status", c.ns, mCAD), &v1alpha1.MCAD{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.MCAD), err +} + +// Delete takes name of the mCAD and deletes it. Returns an error if one occurs. +func (c *FakeMCADs) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteActionWithOptions(mcadsResource, c.ns, name, opts), &v1alpha1.MCAD{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeMCADs) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionAction(mcadsResource, c.ns, listOpts) + + _, err := c.Fake.Invokes(action, &v1alpha1.MCADList{}) + return err +} + +// Patch applies the patch and returns the patched mCAD. +func (c *FakeMCADs) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.MCAD, err error) { + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceAction(mcadsResource, c.ns, name, pt, data, subresources...), &v1alpha1.MCAD{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.MCAD), err +} + +// Apply takes the given apply declarative configuration, applies it and returns the applied mCAD. +func (c *FakeMCADs) Apply(ctx context.Context, mCAD *codeflarev1alpha1.MCADApplyConfiguration, opts v1.ApplyOptions) (result *v1alpha1.MCAD, err error) { + if mCAD == nil { + return nil, fmt.Errorf("mCAD provided to Apply must not be nil") + } + data, err := json.Marshal(mCAD) + if err != nil { + return nil, err + } + name := mCAD.Name + if name == nil { + return nil, fmt.Errorf("mCAD.Name must be provided to Apply") + } + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceAction(mcadsResource, c.ns, *name, types.ApplyPatchType, data), &v1alpha1.MCAD{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.MCAD), err +} + +// ApplyStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating ApplyStatus(). +func (c *FakeMCADs) ApplyStatus(ctx context.Context, mCAD *codeflarev1alpha1.MCADApplyConfiguration, opts v1.ApplyOptions) (result *v1alpha1.MCAD, err error) { + if mCAD == nil { + return nil, fmt.Errorf("mCAD provided to Apply must not be nil") + } + data, err := json.Marshal(mCAD) + if err != nil { + return nil, err + } + name := mCAD.Name + if name == nil { + return nil, fmt.Errorf("mCAD.Name must be provided to Apply") + } + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceAction(mcadsResource, c.ns, *name, types.ApplyPatchType, data, "status"), &v1alpha1.MCAD{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.MCAD), err +} diff --git a/client/clientset/versioned/typed/codeflare/v1alpha1/generated_expansion.go b/client/clientset/versioned/typed/codeflare/v1alpha1/generated_expansion.go new file mode 100644 index 000000000..4727cc4df --- /dev/null +++ b/client/clientset/versioned/typed/codeflare/v1alpha1/generated_expansion.go @@ -0,0 +1,23 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +type InstaScaleExpansion interface{} + +type MCADExpansion interface{} diff --git a/client/clientset/versioned/typed/codeflare/v1alpha1/instascale.go b/client/clientset/versioned/typed/codeflare/v1alpha1/instascale.go new file mode 100644 index 000000000..c146a98d2 --- /dev/null +++ b/client/clientset/versioned/typed/codeflare/v1alpha1/instascale.go @@ -0,0 +1,256 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + json "encoding/json" + "fmt" + "time" + + v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/applyconfiguration/codeflare/v1alpha1" + scheme "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// InstaScalesGetter has a method to return a InstaScaleInterface. +// A group's client should implement this interface. +type InstaScalesGetter interface { + InstaScales(namespace string) InstaScaleInterface +} + +// InstaScaleInterface has methods to work with InstaScale resources. +type InstaScaleInterface interface { + Create(ctx context.Context, instaScale *v1alpha1.InstaScale, opts v1.CreateOptions) (*v1alpha1.InstaScale, error) + Update(ctx context.Context, instaScale *v1alpha1.InstaScale, opts v1.UpdateOptions) (*v1alpha1.InstaScale, error) + UpdateStatus(ctx context.Context, instaScale *v1alpha1.InstaScale, opts v1.UpdateOptions) (*v1alpha1.InstaScale, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.InstaScale, error) + List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.InstaScaleList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.InstaScale, err error) + Apply(ctx context.Context, instaScale *codeflarev1alpha1.InstaScaleApplyConfiguration, opts v1.ApplyOptions) (result *v1alpha1.InstaScale, err error) + ApplyStatus(ctx context.Context, instaScale *codeflarev1alpha1.InstaScaleApplyConfiguration, opts v1.ApplyOptions) (result *v1alpha1.InstaScale, err error) + InstaScaleExpansion +} + +// instaScales implements InstaScaleInterface +type instaScales struct { + client rest.Interface + ns string +} + +// newInstaScales returns a InstaScales +func newInstaScales(c *CodeflareV1alpha1Client, namespace string) *instaScales { + return &instaScales{ + client: c.RESTClient(), + ns: namespace, + } +} + +// Get takes name of the instaScale, and returns the corresponding instaScale object, and an error if there is any. +func (c *instaScales) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.InstaScale, err error) { + result = &v1alpha1.InstaScale{} + err = c.client.Get(). + Namespace(c.ns). + Resource("instascales"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of InstaScales that match those selectors. +func (c *instaScales) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.InstaScaleList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + result = &v1alpha1.InstaScaleList{} + err = c.client.Get(). + Namespace(c.ns). + Resource("instascales"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Do(ctx). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested instaScales. +func (c *instaScales) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + opts.Watch = true + return c.client.Get(). + Namespace(c.ns). + Resource("instascales"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Watch(ctx) +} + +// Create takes the representation of a instaScale and creates it. Returns the server's representation of the instaScale, and an error, if there is any. +func (c *instaScales) Create(ctx context.Context, instaScale *v1alpha1.InstaScale, opts v1.CreateOptions) (result *v1alpha1.InstaScale, err error) { + result = &v1alpha1.InstaScale{} + err = c.client.Post(). + Namespace(c.ns). + Resource("instascales"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(instaScale). + Do(ctx). + Into(result) + return +} + +// Update takes the representation of a instaScale and updates it. Returns the server's representation of the instaScale, and an error, if there is any. +func (c *instaScales) Update(ctx context.Context, instaScale *v1alpha1.InstaScale, opts v1.UpdateOptions) (result *v1alpha1.InstaScale, err error) { + result = &v1alpha1.InstaScale{} + err = c.client.Put(). + Namespace(c.ns). + Resource("instascales"). + Name(instaScale.Name). + VersionedParams(&opts, scheme.ParameterCodec). + Body(instaScale). + Do(ctx). + Into(result) + return +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *instaScales) UpdateStatus(ctx context.Context, instaScale *v1alpha1.InstaScale, opts v1.UpdateOptions) (result *v1alpha1.InstaScale, err error) { + result = &v1alpha1.InstaScale{} + err = c.client.Put(). + Namespace(c.ns). + Resource("instascales"). + Name(instaScale.Name). + SubResource("status"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(instaScale). + Do(ctx). + Into(result) + return +} + +// Delete takes name of the instaScale and deletes it. Returns an error if one occurs. +func (c *instaScales) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + return c.client.Delete(). + Namespace(c.ns). + Resource("instascales"). + Name(name). + Body(&opts). + Do(ctx). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *instaScales) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + var timeout time.Duration + if listOpts.TimeoutSeconds != nil { + timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second + } + return c.client.Delete(). + Namespace(c.ns). + Resource("instascales"). + VersionedParams(&listOpts, scheme.ParameterCodec). + Timeout(timeout). + Body(&opts). + Do(ctx). + Error() +} + +// Patch applies the patch and returns the patched instaScale. +func (c *instaScales) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.InstaScale, err error) { + result = &v1alpha1.InstaScale{} + err = c.client.Patch(pt). + Namespace(c.ns). + Resource("instascales"). + Name(name). + SubResource(subresources...). + VersionedParams(&opts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} + +// Apply takes the given apply declarative configuration, applies it and returns the applied instaScale. +func (c *instaScales) Apply(ctx context.Context, instaScale *codeflarev1alpha1.InstaScaleApplyConfiguration, opts v1.ApplyOptions) (result *v1alpha1.InstaScale, err error) { + if instaScale == nil { + return nil, fmt.Errorf("instaScale provided to Apply must not be nil") + } + patchOpts := opts.ToPatchOptions() + data, err := json.Marshal(instaScale) + if err != nil { + return nil, err + } + name := instaScale.Name + if name == nil { + return nil, fmt.Errorf("instaScale.Name must be provided to Apply") + } + result = &v1alpha1.InstaScale{} + err = c.client.Patch(types.ApplyPatchType). + Namespace(c.ns). + Resource("instascales"). + Name(*name). + VersionedParams(&patchOpts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} + +// ApplyStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating ApplyStatus(). +func (c *instaScales) ApplyStatus(ctx context.Context, instaScale *codeflarev1alpha1.InstaScaleApplyConfiguration, opts v1.ApplyOptions) (result *v1alpha1.InstaScale, err error) { + if instaScale == nil { + return nil, fmt.Errorf("instaScale provided to Apply must not be nil") + } + patchOpts := opts.ToPatchOptions() + data, err := json.Marshal(instaScale) + if err != nil { + return nil, err + } + + name := instaScale.Name + if name == nil { + return nil, fmt.Errorf("instaScale.Name must be provided to Apply") + } + + result = &v1alpha1.InstaScale{} + err = c.client.Patch(types.ApplyPatchType). + Namespace(c.ns). + Resource("instascales"). + Name(*name). + SubResource("status"). + VersionedParams(&patchOpts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} diff --git a/client/clientset/versioned/typed/codeflare/v1alpha1/mcad.go b/client/clientset/versioned/typed/codeflare/v1alpha1/mcad.go new file mode 100644 index 000000000..7d70e4035 --- /dev/null +++ b/client/clientset/versioned/typed/codeflare/v1alpha1/mcad.go @@ -0,0 +1,256 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + json "encoding/json" + "fmt" + "time" + + v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/applyconfiguration/codeflare/v1alpha1" + scheme "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// MCADsGetter has a method to return a MCADInterface. +// A group's client should implement this interface. +type MCADsGetter interface { + MCADs(namespace string) MCADInterface +} + +// MCADInterface has methods to work with MCAD resources. +type MCADInterface interface { + Create(ctx context.Context, mCAD *v1alpha1.MCAD, opts v1.CreateOptions) (*v1alpha1.MCAD, error) + Update(ctx context.Context, mCAD *v1alpha1.MCAD, opts v1.UpdateOptions) (*v1alpha1.MCAD, error) + UpdateStatus(ctx context.Context, mCAD *v1alpha1.MCAD, opts v1.UpdateOptions) (*v1alpha1.MCAD, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.MCAD, error) + List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.MCADList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.MCAD, err error) + Apply(ctx context.Context, mCAD *codeflarev1alpha1.MCADApplyConfiguration, opts v1.ApplyOptions) (result *v1alpha1.MCAD, err error) + ApplyStatus(ctx context.Context, mCAD *codeflarev1alpha1.MCADApplyConfiguration, opts v1.ApplyOptions) (result *v1alpha1.MCAD, err error) + MCADExpansion +} + +// mCADs implements MCADInterface +type mCADs struct { + client rest.Interface + ns string +} + +// newMCADs returns a MCADs +func newMCADs(c *CodeflareV1alpha1Client, namespace string) *mCADs { + return &mCADs{ + client: c.RESTClient(), + ns: namespace, + } +} + +// Get takes name of the mCAD, and returns the corresponding mCAD object, and an error if there is any. +func (c *mCADs) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.MCAD, err error) { + result = &v1alpha1.MCAD{} + err = c.client.Get(). + Namespace(c.ns). + Resource("mcads"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of MCADs that match those selectors. +func (c *mCADs) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.MCADList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + result = &v1alpha1.MCADList{} + err = c.client.Get(). + Namespace(c.ns). + Resource("mcads"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Do(ctx). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested mCADs. +func (c *mCADs) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + opts.Watch = true + return c.client.Get(). + Namespace(c.ns). + Resource("mcads"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Watch(ctx) +} + +// Create takes the representation of a mCAD and creates it. Returns the server's representation of the mCAD, and an error, if there is any. +func (c *mCADs) Create(ctx context.Context, mCAD *v1alpha1.MCAD, opts v1.CreateOptions) (result *v1alpha1.MCAD, err error) { + result = &v1alpha1.MCAD{} + err = c.client.Post(). + Namespace(c.ns). + Resource("mcads"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(mCAD). + Do(ctx). + Into(result) + return +} + +// Update takes the representation of a mCAD and updates it. Returns the server's representation of the mCAD, and an error, if there is any. +func (c *mCADs) Update(ctx context.Context, mCAD *v1alpha1.MCAD, opts v1.UpdateOptions) (result *v1alpha1.MCAD, err error) { + result = &v1alpha1.MCAD{} + err = c.client.Put(). + Namespace(c.ns). + Resource("mcads"). + Name(mCAD.Name). + VersionedParams(&opts, scheme.ParameterCodec). + Body(mCAD). + Do(ctx). + Into(result) + return +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *mCADs) UpdateStatus(ctx context.Context, mCAD *v1alpha1.MCAD, opts v1.UpdateOptions) (result *v1alpha1.MCAD, err error) { + result = &v1alpha1.MCAD{} + err = c.client.Put(). + Namespace(c.ns). + Resource("mcads"). + Name(mCAD.Name). + SubResource("status"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(mCAD). + Do(ctx). + Into(result) + return +} + +// Delete takes name of the mCAD and deletes it. Returns an error if one occurs. +func (c *mCADs) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + return c.client.Delete(). + Namespace(c.ns). + Resource("mcads"). + Name(name). + Body(&opts). + Do(ctx). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *mCADs) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + var timeout time.Duration + if listOpts.TimeoutSeconds != nil { + timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second + } + return c.client.Delete(). + Namespace(c.ns). + Resource("mcads"). + VersionedParams(&listOpts, scheme.ParameterCodec). + Timeout(timeout). + Body(&opts). + Do(ctx). + Error() +} + +// Patch applies the patch and returns the patched mCAD. +func (c *mCADs) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.MCAD, err error) { + result = &v1alpha1.MCAD{} + err = c.client.Patch(pt). + Namespace(c.ns). + Resource("mcads"). + Name(name). + SubResource(subresources...). + VersionedParams(&opts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} + +// Apply takes the given apply declarative configuration, applies it and returns the applied mCAD. +func (c *mCADs) Apply(ctx context.Context, mCAD *codeflarev1alpha1.MCADApplyConfiguration, opts v1.ApplyOptions) (result *v1alpha1.MCAD, err error) { + if mCAD == nil { + return nil, fmt.Errorf("mCAD provided to Apply must not be nil") + } + patchOpts := opts.ToPatchOptions() + data, err := json.Marshal(mCAD) + if err != nil { + return nil, err + } + name := mCAD.Name + if name == nil { + return nil, fmt.Errorf("mCAD.Name must be provided to Apply") + } + result = &v1alpha1.MCAD{} + err = c.client.Patch(types.ApplyPatchType). + Namespace(c.ns). + Resource("mcads"). + Name(*name). + VersionedParams(&patchOpts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} + +// ApplyStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating ApplyStatus(). +func (c *mCADs) ApplyStatus(ctx context.Context, mCAD *codeflarev1alpha1.MCADApplyConfiguration, opts v1.ApplyOptions) (result *v1alpha1.MCAD, err error) { + if mCAD == nil { + return nil, fmt.Errorf("mCAD provided to Apply must not be nil") + } + patchOpts := opts.ToPatchOptions() + data, err := json.Marshal(mCAD) + if err != nil { + return nil, err + } + + name := mCAD.Name + if name == nil { + return nil, fmt.Errorf("mCAD.Name must be provided to Apply") + } + + result = &v1alpha1.MCAD{} + err = c.client.Patch(types.ApplyPatchType). + Namespace(c.ns). + Resource("mcads"). + Name(*name). + SubResource("status"). + VersionedParams(&patchOpts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} diff --git a/client/informer/externalversions/codeflare/interface.go b/client/informer/externalversions/codeflare/interface.go new file mode 100644 index 000000000..63ab10ffc --- /dev/null +++ b/client/informer/externalversions/codeflare/interface.go @@ -0,0 +1,46 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package codeflare + +import ( + v1alpha1 "github.com/project-codeflare/codeflare-operator/client/informer/externalversions/codeflare/v1alpha1" + internalinterfaces "github.com/project-codeflare/codeflare-operator/client/informer/externalversions/internalinterfaces" +) + +// Interface provides access to each of this group's versions. +type Interface interface { + // V1alpha1 provides access to shared informers for resources in V1alpha1. + V1alpha1() v1alpha1.Interface +} + +type group struct { + factory internalinterfaces.SharedInformerFactory + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// New returns a new Interface. +func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { + return &group{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} +} + +// V1alpha1 returns a new v1alpha1.Interface. +func (g *group) V1alpha1() v1alpha1.Interface { + return v1alpha1.New(g.factory, g.namespace, g.tweakListOptions) +} diff --git a/client/informer/externalversions/codeflare/v1alpha1/instascale.go b/client/informer/externalversions/codeflare/v1alpha1/instascale.go new file mode 100644 index 000000000..70346e653 --- /dev/null +++ b/client/informer/externalversions/codeflare/v1alpha1/instascale.go @@ -0,0 +1,90 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + time "time" + + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" + versioned "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" + internalinterfaces "github.com/project-codeflare/codeflare-operator/client/informer/externalversions/internalinterfaces" + v1alpha1 "github.com/project-codeflare/codeflare-operator/client/listers/codeflare/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// InstaScaleInformer provides access to a shared informer and lister for +// InstaScales. +type InstaScaleInformer interface { + Informer() cache.SharedIndexInformer + Lister() v1alpha1.InstaScaleLister +} + +type instaScaleInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewInstaScaleInformer constructs a new informer for InstaScale type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewInstaScaleInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredInstaScaleInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredInstaScaleInformer constructs a new informer for InstaScale type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredInstaScaleInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.CodeflareV1alpha1().InstaScales(namespace).List(context.TODO(), options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.CodeflareV1alpha1().InstaScales(namespace).Watch(context.TODO(), options) + }, + }, + &codeflarev1alpha1.InstaScale{}, + resyncPeriod, + indexers, + ) +} + +func (f *instaScaleInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredInstaScaleInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *instaScaleInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&codeflarev1alpha1.InstaScale{}, f.defaultInformer) +} + +func (f *instaScaleInformer) Lister() v1alpha1.InstaScaleLister { + return v1alpha1.NewInstaScaleLister(f.Informer().GetIndexer()) +} diff --git a/client/informer/externalversions/codeflare/v1alpha1/interface.go b/client/informer/externalversions/codeflare/v1alpha1/interface.go new file mode 100644 index 000000000..6003a7031 --- /dev/null +++ b/client/informer/externalversions/codeflare/v1alpha1/interface.go @@ -0,0 +1,52 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + internalinterfaces "github.com/project-codeflare/codeflare-operator/client/informer/externalversions/internalinterfaces" +) + +// Interface provides access to all the informers in this group version. +type Interface interface { + // InstaScales returns a InstaScaleInformer. + InstaScales() InstaScaleInformer + // MCADs returns a MCADInformer. + MCADs() MCADInformer +} + +type version struct { + factory internalinterfaces.SharedInformerFactory + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// New returns a new Interface. +func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { + return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} +} + +// InstaScales returns a InstaScaleInformer. +func (v *version) InstaScales() InstaScaleInformer { + return &instaScaleInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} + +// MCADs returns a MCADInformer. +func (v *version) MCADs() MCADInformer { + return &mCADInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} diff --git a/client/informer/externalversions/codeflare/v1alpha1/mcad.go b/client/informer/externalversions/codeflare/v1alpha1/mcad.go new file mode 100644 index 000000000..2fe4d5a30 --- /dev/null +++ b/client/informer/externalversions/codeflare/v1alpha1/mcad.go @@ -0,0 +1,90 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + time "time" + + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" + versioned "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" + internalinterfaces "github.com/project-codeflare/codeflare-operator/client/informer/externalversions/internalinterfaces" + v1alpha1 "github.com/project-codeflare/codeflare-operator/client/listers/codeflare/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// MCADInformer provides access to a shared informer and lister for +// MCADs. +type MCADInformer interface { + Informer() cache.SharedIndexInformer + Lister() v1alpha1.MCADLister +} + +type mCADInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewMCADInformer constructs a new informer for MCAD type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewMCADInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredMCADInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredMCADInformer constructs a new informer for MCAD type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredMCADInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.CodeflareV1alpha1().MCADs(namespace).List(context.TODO(), options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.CodeflareV1alpha1().MCADs(namespace).Watch(context.TODO(), options) + }, + }, + &codeflarev1alpha1.MCAD{}, + resyncPeriod, + indexers, + ) +} + +func (f *mCADInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredMCADInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *mCADInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&codeflarev1alpha1.MCAD{}, f.defaultInformer) +} + +func (f *mCADInformer) Lister() v1alpha1.MCADLister { + return v1alpha1.NewMCADLister(f.Informer().GetIndexer()) +} diff --git a/client/informer/externalversions/factory.go b/client/informer/externalversions/factory.go new file mode 100644 index 000000000..e1f7f14a1 --- /dev/null +++ b/client/informer/externalversions/factory.go @@ -0,0 +1,251 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package externalversions + +import ( + reflect "reflect" + sync "sync" + time "time" + + versioned "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" + codeflare "github.com/project-codeflare/codeflare-operator/client/informer/externalversions/codeflare" + internalinterfaces "github.com/project-codeflare/codeflare-operator/client/informer/externalversions/internalinterfaces" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + cache "k8s.io/client-go/tools/cache" +) + +// SharedInformerOption defines the functional option type for SharedInformerFactory. +type SharedInformerOption func(*sharedInformerFactory) *sharedInformerFactory + +type sharedInformerFactory struct { + client versioned.Interface + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc + lock sync.Mutex + defaultResync time.Duration + customResync map[reflect.Type]time.Duration + + informers map[reflect.Type]cache.SharedIndexInformer + // startedInformers is used for tracking which informers have been started. + // This allows Start() to be called multiple times safely. + startedInformers map[reflect.Type]bool + // wg tracks how many goroutines were started. + wg sync.WaitGroup + // shuttingDown is true when Shutdown has been called. It may still be running + // because it needs to wait for goroutines. + shuttingDown bool +} + +// WithCustomResyncConfig sets a custom resync period for the specified informer types. +func WithCustomResyncConfig(resyncConfig map[v1.Object]time.Duration) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + for k, v := range resyncConfig { + factory.customResync[reflect.TypeOf(k)] = v + } + return factory + } +} + +// WithTweakListOptions sets a custom filter on all listers of the configured SharedInformerFactory. +func WithTweakListOptions(tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + factory.tweakListOptions = tweakListOptions + return factory + } +} + +// WithNamespace limits the SharedInformerFactory to the specified namespace. +func WithNamespace(namespace string) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + factory.namespace = namespace + return factory + } +} + +// NewSharedInformerFactory constructs a new instance of sharedInformerFactory for all namespaces. +func NewSharedInformerFactory(client versioned.Interface, defaultResync time.Duration) SharedInformerFactory { + return NewSharedInformerFactoryWithOptions(client, defaultResync) +} + +// NewFilteredSharedInformerFactory constructs a new instance of sharedInformerFactory. +// Listers obtained via this SharedInformerFactory will be subject to the same filters +// as specified here. +// Deprecated: Please use NewSharedInformerFactoryWithOptions instead +func NewFilteredSharedInformerFactory(client versioned.Interface, defaultResync time.Duration, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerFactory { + return NewSharedInformerFactoryWithOptions(client, defaultResync, WithNamespace(namespace), WithTweakListOptions(tweakListOptions)) +} + +// NewSharedInformerFactoryWithOptions constructs a new instance of a SharedInformerFactory with additional options. +func NewSharedInformerFactoryWithOptions(client versioned.Interface, defaultResync time.Duration, options ...SharedInformerOption) SharedInformerFactory { + factory := &sharedInformerFactory{ + client: client, + namespace: v1.NamespaceAll, + defaultResync: defaultResync, + informers: make(map[reflect.Type]cache.SharedIndexInformer), + startedInformers: make(map[reflect.Type]bool), + customResync: make(map[reflect.Type]time.Duration), + } + + // Apply all options + for _, opt := range options { + factory = opt(factory) + } + + return factory +} + +func (f *sharedInformerFactory) Start(stopCh <-chan struct{}) { + f.lock.Lock() + defer f.lock.Unlock() + + if f.shuttingDown { + return + } + + for informerType, informer := range f.informers { + if !f.startedInformers[informerType] { + f.wg.Add(1) + // We need a new variable in each loop iteration, + // otherwise the goroutine would use the loop variable + // and that keeps changing. + informer := informer + go func() { + defer f.wg.Done() + informer.Run(stopCh) + }() + f.startedInformers[informerType] = true + } + } +} + +func (f *sharedInformerFactory) Shutdown() { + f.lock.Lock() + f.shuttingDown = true + f.lock.Unlock() + + // Will return immediately if there is nothing to wait for. + f.wg.Wait() +} + +func (f *sharedInformerFactory) WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool { + informers := func() map[reflect.Type]cache.SharedIndexInformer { + f.lock.Lock() + defer f.lock.Unlock() + + informers := map[reflect.Type]cache.SharedIndexInformer{} + for informerType, informer := range f.informers { + if f.startedInformers[informerType] { + informers[informerType] = informer + } + } + return informers + }() + + res := map[reflect.Type]bool{} + for informType, informer := range informers { + res[informType] = cache.WaitForCacheSync(stopCh, informer.HasSynced) + } + return res +} + +// InternalInformerFor returns the SharedIndexInformer for obj using an internal +// client. +func (f *sharedInformerFactory) InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer { + f.lock.Lock() + defer f.lock.Unlock() + + informerType := reflect.TypeOf(obj) + informer, exists := f.informers[informerType] + if exists { + return informer + } + + resyncPeriod, exists := f.customResync[informerType] + if !exists { + resyncPeriod = f.defaultResync + } + + informer = newFunc(f.client, resyncPeriod) + f.informers[informerType] = informer + + return informer +} + +// SharedInformerFactory provides shared informers for resources in all known +// API group versions. +// +// It is typically used like this: +// +// ctx, cancel := context.Background() +// defer cancel() +// factory := NewSharedInformerFactory(client, resyncPeriod) +// defer factory.WaitForStop() // Returns immediately if nothing was started. +// genericInformer := factory.ForResource(resource) +// typedInformer := factory.SomeAPIGroup().V1().SomeType() +// factory.Start(ctx.Done()) // Start processing these informers. +// synced := factory.WaitForCacheSync(ctx.Done()) +// for v, ok := range synced { +// if !ok { +// fmt.Fprintf(os.Stderr, "caches failed to sync: %v", v) +// return +// } +// } +// +// // Creating informers can also be created after Start, but then +// // Start must be called again: +// anotherGenericInformer := factory.ForResource(resource) +// factory.Start(ctx.Done()) +type SharedInformerFactory interface { + internalinterfaces.SharedInformerFactory + + // Start initializes all requested informers. They are handled in goroutines + // which run until the stop channel gets closed. + Start(stopCh <-chan struct{}) + + // Shutdown marks a factory as shutting down. At that point no new + // informers can be started anymore and Start will return without + // doing anything. + // + // In addition, Shutdown blocks until all goroutines have terminated. For that + // to happen, the close channel(s) that they were started with must be closed, + // either before Shutdown gets called or while it is waiting. + // + // Shutdown may be called multiple times, even concurrently. All such calls will + // block until all goroutines have terminated. + Shutdown() + + // WaitForCacheSync blocks until all started informers' caches were synced + // or the stop channel gets closed. + WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool + + // ForResource gives generic access to a shared informer of the matching type. + ForResource(resource schema.GroupVersionResource) (GenericInformer, error) + + // InternalInformerFor returns the SharedIndexInformer for obj using an internal + // client. + InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer + + Codeflare() codeflare.Interface +} + +func (f *sharedInformerFactory) Codeflare() codeflare.Interface { + return codeflare.New(f, f.namespace, f.tweakListOptions) +} diff --git a/client/informer/externalversions/generic.go b/client/informer/externalversions/generic.go new file mode 100644 index 000000000..9eea54ebf --- /dev/null +++ b/client/informer/externalversions/generic.go @@ -0,0 +1,64 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package externalversions + +import ( + "fmt" + + v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" + schema "k8s.io/apimachinery/pkg/runtime/schema" + cache "k8s.io/client-go/tools/cache" +) + +// GenericInformer is type of SharedIndexInformer which will locate and delegate to other +// sharedInformers based on type +type GenericInformer interface { + Informer() cache.SharedIndexInformer + Lister() cache.GenericLister +} + +type genericInformer struct { + informer cache.SharedIndexInformer + resource schema.GroupResource +} + +// Informer returns the SharedIndexInformer. +func (f *genericInformer) Informer() cache.SharedIndexInformer { + return f.informer +} + +// Lister returns the GenericLister. +func (f *genericInformer) Lister() cache.GenericLister { + return cache.NewGenericLister(f.Informer().GetIndexer(), f.resource) +} + +// ForResource gives generic access to a shared informer of the matching type +// TODO extend this to unknown resources with a client pool +func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) { + switch resource { + // Group=codeflare.codeflare.dev, Version=v1alpha1 + case v1alpha1.SchemeGroupVersion.WithResource("instascales"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Codeflare().V1alpha1().InstaScales().Informer()}, nil + case v1alpha1.SchemeGroupVersion.WithResource("mcads"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Codeflare().V1alpha1().MCADs().Informer()}, nil + + } + + return nil, fmt.Errorf("no informer found for %v", resource) +} diff --git a/client/informer/externalversions/internalinterfaces/factory_interfaces.go b/client/informer/externalversions/internalinterfaces/factory_interfaces.go new file mode 100644 index 000000000..b380f668c --- /dev/null +++ b/client/informer/externalversions/internalinterfaces/factory_interfaces.go @@ -0,0 +1,40 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package internalinterfaces + +import ( + time "time" + + versioned "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + cache "k8s.io/client-go/tools/cache" +) + +// NewInformerFunc takes versioned.Interface and time.Duration to return a SharedIndexInformer. +type NewInformerFunc func(versioned.Interface, time.Duration) cache.SharedIndexInformer + +// SharedInformerFactory a small interface to allow for adding an informer without an import cycle +type SharedInformerFactory interface { + Start(stopCh <-chan struct{}) + InformerFor(obj runtime.Object, newFunc NewInformerFunc) cache.SharedIndexInformer +} + +// TweakListOptionsFunc is a function that transforms a v1.ListOptions. +type TweakListOptionsFunc func(*v1.ListOptions) diff --git a/client/listers/codeflare/v1alpha1/expansion_generated.go b/client/listers/codeflare/v1alpha1/expansion_generated.go new file mode 100644 index 000000000..23082ab9d --- /dev/null +++ b/client/listers/codeflare/v1alpha1/expansion_generated.go @@ -0,0 +1,35 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1alpha1 + +// InstaScaleListerExpansion allows custom methods to be added to +// InstaScaleLister. +type InstaScaleListerExpansion interface{} + +// InstaScaleNamespaceListerExpansion allows custom methods to be added to +// InstaScaleNamespaceLister. +type InstaScaleNamespaceListerExpansion interface{} + +// MCADListerExpansion allows custom methods to be added to +// MCADLister. +type MCADListerExpansion interface{} + +// MCADNamespaceListerExpansion allows custom methods to be added to +// MCADNamespaceLister. +type MCADNamespaceListerExpansion interface{} diff --git a/client/listers/codeflare/v1alpha1/instascale.go b/client/listers/codeflare/v1alpha1/instascale.go new file mode 100644 index 000000000..ee5c22847 --- /dev/null +++ b/client/listers/codeflare/v1alpha1/instascale.go @@ -0,0 +1,99 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/cache" +) + +// InstaScaleLister helps list InstaScales. +// All objects returned here must be treated as read-only. +type InstaScaleLister interface { + // List lists all InstaScales in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1alpha1.InstaScale, err error) + // InstaScales returns an object that can list and get InstaScales. + InstaScales(namespace string) InstaScaleNamespaceLister + InstaScaleListerExpansion +} + +// instaScaleLister implements the InstaScaleLister interface. +type instaScaleLister struct { + indexer cache.Indexer +} + +// NewInstaScaleLister returns a new InstaScaleLister. +func NewInstaScaleLister(indexer cache.Indexer) InstaScaleLister { + return &instaScaleLister{indexer: indexer} +} + +// List lists all InstaScales in the indexer. +func (s *instaScaleLister) List(selector labels.Selector) (ret []*v1alpha1.InstaScale, err error) { + err = cache.ListAll(s.indexer, selector, func(m interface{}) { + ret = append(ret, m.(*v1alpha1.InstaScale)) + }) + return ret, err +} + +// InstaScales returns an object that can list and get InstaScales. +func (s *instaScaleLister) InstaScales(namespace string) InstaScaleNamespaceLister { + return instaScaleNamespaceLister{indexer: s.indexer, namespace: namespace} +} + +// InstaScaleNamespaceLister helps list and get InstaScales. +// All objects returned here must be treated as read-only. +type InstaScaleNamespaceLister interface { + // List lists all InstaScales in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1alpha1.InstaScale, err error) + // Get retrieves the InstaScale from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*v1alpha1.InstaScale, error) + InstaScaleNamespaceListerExpansion +} + +// instaScaleNamespaceLister implements the InstaScaleNamespaceLister +// interface. +type instaScaleNamespaceLister struct { + indexer cache.Indexer + namespace string +} + +// List lists all InstaScales in the indexer for a given namespace. +func (s instaScaleNamespaceLister) List(selector labels.Selector) (ret []*v1alpha1.InstaScale, err error) { + err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { + ret = append(ret, m.(*v1alpha1.InstaScale)) + }) + return ret, err +} + +// Get retrieves the InstaScale from the indexer for a given namespace and name. +func (s instaScaleNamespaceLister) Get(name string) (*v1alpha1.InstaScale, error) { + obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.NewNotFound(v1alpha1.Resource("instascale"), name) + } + return obj.(*v1alpha1.InstaScale), nil +} diff --git a/client/listers/codeflare/v1alpha1/mcad.go b/client/listers/codeflare/v1alpha1/mcad.go new file mode 100644 index 000000000..8ac44bfce --- /dev/null +++ b/client/listers/codeflare/v1alpha1/mcad.go @@ -0,0 +1,99 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/cache" +) + +// MCADLister helps list MCADs. +// All objects returned here must be treated as read-only. +type MCADLister interface { + // List lists all MCADs in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1alpha1.MCAD, err error) + // MCADs returns an object that can list and get MCADs. + MCADs(namespace string) MCADNamespaceLister + MCADListerExpansion +} + +// mCADLister implements the MCADLister interface. +type mCADLister struct { + indexer cache.Indexer +} + +// NewMCADLister returns a new MCADLister. +func NewMCADLister(indexer cache.Indexer) MCADLister { + return &mCADLister{indexer: indexer} +} + +// List lists all MCADs in the indexer. +func (s *mCADLister) List(selector labels.Selector) (ret []*v1alpha1.MCAD, err error) { + err = cache.ListAll(s.indexer, selector, func(m interface{}) { + ret = append(ret, m.(*v1alpha1.MCAD)) + }) + return ret, err +} + +// MCADs returns an object that can list and get MCADs. +func (s *mCADLister) MCADs(namespace string) MCADNamespaceLister { + return mCADNamespaceLister{indexer: s.indexer, namespace: namespace} +} + +// MCADNamespaceLister helps list and get MCADs. +// All objects returned here must be treated as read-only. +type MCADNamespaceLister interface { + // List lists all MCADs in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1alpha1.MCAD, err error) + // Get retrieves the MCAD from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*v1alpha1.MCAD, error) + MCADNamespaceListerExpansion +} + +// mCADNamespaceLister implements the MCADNamespaceLister +// interface. +type mCADNamespaceLister struct { + indexer cache.Indexer + namespace string +} + +// List lists all MCADs in the indexer for a given namespace. +func (s mCADNamespaceLister) List(selector labels.Selector) (ret []*v1alpha1.MCAD, err error) { + err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { + ret = append(ret, m.(*v1alpha1.MCAD)) + }) + return ret, err +} + +// Get retrieves the MCAD from the indexer for a given namespace and name. +func (s mCADNamespaceLister) Get(name string) (*v1alpha1.MCAD, error) { + obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.NewNotFound(v1alpha1.Resource("mcad"), name) + } + return obj.(*v1alpha1.MCAD), nil +} diff --git a/controllers/instascale.go b/controllers/instascale.go index f4b9c3d3f..3a35e05a9 100644 --- a/controllers/instascale.go +++ b/controllers/instascale.go @@ -1,7 +1,7 @@ package controllers import ( - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/v1alpha1" + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" ) var instascaleTemplates = []string{ diff --git a/controllers/instascale_controller.go b/controllers/instascale_controller.go index c9fabdf57..31fcb53b9 100644 --- a/controllers/instascale_controller.go +++ b/controllers/instascale_controller.go @@ -39,7 +39,7 @@ import ( "github.com/go-logr/logr" mf "github.com/manifestival/manifestival" - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/v1alpha1" + "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" "github.com/project-codeflare/codeflare-operator/controllers/config" "github.com/project-codeflare/codeflare-operator/controllers/util" ) @@ -106,7 +106,7 @@ func (r *InstaScaleReconciler) Reconcile(ctx context.Context, req ctrl.Request) log.V(1).Info("InstaScale reconciler called.") params := &InstaScaleParams{} - instascaleCustomResource := &codeflarev1alpha1.InstaScale{} + instascaleCustomResource := &v1alpha1.InstaScale{} err := r.Get(ctx, req.NamespacedName, instascaleCustomResource) if err != nil && apierrs.IsNotFound(err) { @@ -122,7 +122,7 @@ func (r *InstaScaleReconciler) Reconcile(ctx context.Context, req ctrl.Request) // In production we expect these to be populated if instascaleCustomResource.Kind == "" { instascaleCustomResource = instascaleCustomResource.DeepCopy() - gvk := codeflarev1alpha1.GroupVersion.WithKind("InstaScale") + gvk := v1alpha1.SchemeGroupVersion.WithKind("InstaScale") instascaleCustomResource.APIVersion, instascaleCustomResource.Kind = gvk.Version, gvk.Kind } @@ -168,7 +168,7 @@ func (r *InstaScaleReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } -func updateInstascaleReadyStatus(ctx context.Context, r *InstaScaleReconciler, req ctrl.Request, instascaleCustomResource *codeflarev1alpha1.InstaScale) error { +func updateInstascaleReadyStatus(ctx context.Context, r *InstaScaleReconciler, req ctrl.Request, instascaleCustomResource *v1alpha1.InstaScale) error { deployment := &appsv1.Deployment{} err := r.Get(ctx, types.NamespacedName{Name: fmt.Sprintf("instascale-%s", req.Name), Namespace: req.Namespace}, deployment) if err != nil { @@ -198,7 +198,7 @@ func (r *InstaScaleReconciler) SetupWithManager(mgr ctrl.Manager) error { return nil }) return ctrl.NewControllerManagedBy(mgr). - For(&codeflarev1alpha1.InstaScale{}). + For(&v1alpha1.InstaScale{}). Owns(&corev1.ConfigMap{}). Owns(&corev1.ServiceAccount{}). Owns(&authv1.ClusterRole{}). diff --git a/controllers/instascale_controller_test.go b/controllers/instascale_controller_test.go index 7017e681f..4071c515a 100644 --- a/controllers/instascale_controller_test.go +++ b/controllers/instascale_controller_test.go @@ -9,7 +9,7 @@ import ( mfc "github.com/manifestival/controller-runtime-client" mf "github.com/manifestival/manifestival" - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/v1alpha1" + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" ) const ( diff --git a/controllers/instascale_params.go b/controllers/instascale_params.go index eb6737c79..0def69351 100644 --- a/controllers/instascale_params.go +++ b/controllers/instascale_params.go @@ -8,7 +8,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" - instascalev1alpha1 "github.com/project-codeflare/codeflare-operator/api/v1alpha1" + instascalev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" ) type InstaScaleParams struct { diff --git a/controllers/mcad_controller.go b/controllers/mcad_controller.go index 68dc0886c..1dfe8b9b1 100644 --- a/controllers/mcad_controller.go +++ b/controllers/mcad_controller.go @@ -22,22 +22,24 @@ import ( "github.com/go-logr/logr" mf "github.com/manifestival/manifestival" - "github.com/project-codeflare/codeflare-operator/controllers/config" - "github.com/project-codeflare/codeflare-operator/controllers/util" - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/v1alpha1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" + "github.com/project-codeflare/codeflare-operator/controllers/config" + "github.com/project-codeflare/codeflare-operator/controllers/util" ) const finalizerName = "codeflare.codeflare.dev/finalizer" @@ -98,30 +100,30 @@ func (r *MCADReconciler) DeleteResource(params *MCADParams, template string, fns return tmplManifest.Delete() } -//+kubebuilder:rbac:groups=codeflare.codeflare.dev,resources=mcads,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=codeflare.codeflare.dev,resources=mcads/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=codeflare.codeflare.dev,resources=mcads/finalizers,verbs=update -//+kubebuilder:rbac:groups=mcad.ibm.com,resources=queuejobs;schedulingspecs;appwrappers;appwrappers/finalizers;appwrappers/status,verbs=get;list;watch;create;update;patch;delete;deletecollection -//+kubebuilder:rbac:groups=core,resources=pods;lists;namespaces,verbs=get;list;watch;create;update;patch;delete;deletecollection -//+kubebuilder:rbac:groups=core,resources=bindings;pods/binding,verbs=create -//+kubebuilder:rbac:groups=core,resources=kube-scheduler,verbs=get;update -//+kubebuilder:rbac:groups=core,resources=endpoints;kube-scheduler,verbs=create;get;update -//+kubebuilder:rbac:groups=core,resources=events,verbs=create;patch;update -//+kubebuilder:rbac:groups=core,resources=pods/status,verbs=patch;update -//+kubebuilder:rbac:groups=core,resources=replicationcontrollers,verbs=get;list;watch -//+kubebuilder:rbac:groups=scheduling.sigs.k8s.io,resources=podgroups,verbs=get;list;watch;create;update;patch;delete;deletecollection -//+kubebuilder:rbac:groups=apps,resources=deployments;replicasets;statefulsets,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=*,resources=deployments;services,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=core,resources=secrets;configmaps;services;serviceaccounts;persistentvolumes;persistentvolumeclaims,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=core,resources=persistentvolumes;persistentvolumeclaims,verbs=* -//+kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterroles;clusterrolebindings,verbs=get;list;watch;create;update;delete -//+kubebuilder:rbac:groups=custom.metrics.k8s.io,resources=*,verbs=* -//+kubebuilder:rbac:groups=coordination.k8s.io,resources=leases;kube-scheduler,verbs=create;update;get -//+kubebuilder:rbac:groups=events.k8s.io,resources=events;kube-scheduler,verbs=create;update;patch -//+kubebuilder:rbac:groups=extensions,resources=replicasets,verbs=get;list;watch -//+kubebuilder:rbac:groups=policy,resources=poddisruptionbudgets,verbs=get;list;watch -//+kubebuilder:rbac:groups=storage.k8s.io,resources=csidrivers;csinodes;csistoragecapacities,verbs=get;list;watch +// +kubebuilder:rbac:groups=codeflare.codeflare.dev,resources=mcads,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=codeflare.codeflare.dev,resources=mcads/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=codeflare.codeflare.dev,resources=mcads/finalizers,verbs=update +// +kubebuilder:rbac:groups=mcad.ibm.com,resources=queuejobs;schedulingspecs;appwrappers;appwrappers/finalizers;appwrappers/status,verbs=get;list;watch;create;update;patch;delete;deletecollection +// +kubebuilder:rbac:groups=core,resources=pods;lists;namespaces,verbs=get;list;watch;create;update;patch;delete;deletecollection +// +kubebuilder:rbac:groups=core,resources=bindings;pods/binding,verbs=create +// +kubebuilder:rbac:groups=core,resources=kube-scheduler,verbs=get;update +// +kubebuilder:rbac:groups=core,resources=endpoints;kube-scheduler,verbs=create;get;update +// +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch;update +// +kubebuilder:rbac:groups=core,resources=pods/status,verbs=patch;update +// +kubebuilder:rbac:groups=core,resources=replicationcontrollers,verbs=get;list;watch +// +kubebuilder:rbac:groups=scheduling.sigs.k8s.io,resources=podgroups,verbs=get;list;watch;create;update;patch;delete;deletecollection +// +kubebuilder:rbac:groups=apps,resources=deployments;replicasets;statefulsets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=*,resources=deployments;services,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=secrets;configmaps;services;serviceaccounts;persistentvolumes;persistentvolumeclaims,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=persistentvolumes;persistentvolumeclaims,verbs=* +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterroles;clusterrolebindings,verbs=get;list;watch;create;update;delete +// +kubebuilder:rbac:groups=custom.metrics.k8s.io,resources=*,verbs=* +// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases;kube-scheduler,verbs=create;update;get +// +kubebuilder:rbac:groups=events.k8s.io,resources=events;kube-scheduler,verbs=create;update;patch +// +kubebuilder:rbac:groups=extensions,resources=replicasets,verbs=get;list;watch +// +kubebuilder:rbac:groups=policy,resources=poddisruptionbudgets,verbs=get;list;watch +// +kubebuilder:rbac:groups=storage.k8s.io,resources=csidrivers;csinodes;csistoragecapacities,verbs=get;list;watch func (r *MCADReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.Log.WithValues("namespace", req.Namespace) @@ -129,7 +131,7 @@ func (r *MCADReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. log.V(1).Info("MCAD reconciler called.") params := &MCADParams{} - mcadCustomResource := &codeflarev1alpha1.MCAD{} + mcadCustomResource := &v1alpha1.MCAD{} err := r.Get(ctx, req.NamespacedName, mcadCustomResource) if err != nil && apierrs.IsNotFound(err) { @@ -145,7 +147,7 @@ func (r *MCADReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. // In production we expect these to be populated if mcadCustomResource.Kind == "" { mcadCustomResource = mcadCustomResource.DeepCopy() - gvk := codeflarev1alpha1.GroupVersion.WithKind("MCAD") + gvk := v1alpha1.SchemeGroupVersion.WithKind("MCAD") mcadCustomResource.APIVersion, mcadCustomResource.Kind = gvk.Version, gvk.Kind } @@ -195,7 +197,7 @@ func (r *MCADReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. return ctrl.Result{}, nil } -func updateMCADReadyStatus(ctx context.Context, r *MCADReconciler, req ctrl.Request, mcadCustomResource *codeflarev1alpha1.MCAD) error { +func updateMCADReadyStatus(ctx context.Context, r *MCADReconciler, req ctrl.Request, mcadCustomResource *v1alpha1.MCAD) error { deployment := &appsv1.Deployment{} err := r.Get(ctx, types.NamespacedName{Name: fmt.Sprintf("mcad-controller-%s", req.Name), Namespace: req.Namespace}, deployment) if err != nil { @@ -225,7 +227,7 @@ func (r *MCADReconciler) SetupWithManager(mgr ctrl.Manager) error { return nil }) return ctrl.NewControllerManagedBy(mgr). - For(&codeflarev1alpha1.MCAD{}). + For(&v1alpha1.MCAD{}). Owns(&appsv1.Deployment{}). Owns(&corev1.ConfigMap{}). Owns(&corev1.Service{}). diff --git a/controllers/mcad_controller_test.go b/controllers/mcad_controller_test.go index 69247d3f1..ff1189c4c 100644 --- a/controllers/mcad_controller_test.go +++ b/controllers/mcad_controller_test.go @@ -3,11 +3,13 @@ package controllers import ( "context" - mfc "github.com/manifestival/controller-runtime-client" - mf "github.com/manifestival/manifestival" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/v1alpha1" + + mfc "github.com/manifestival/controller-runtime-client" + mf "github.com/manifestival/manifestival" + + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" ) const ( diff --git a/controllers/mcad_params.go b/controllers/mcad_params.go index ca56523db..cedbdecc7 100644 --- a/controllers/mcad_params.go +++ b/controllers/mcad_params.go @@ -1,8 +1,24 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package controllers import ( mf "github.com/manifestival/manifestival" - mcadv1alpha1 "github.com/project-codeflare/codeflare-operator/api/v1alpha1" + mcadv1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" ) type MCADParams struct { diff --git a/controllers/multi_cluster_app_dispatcher.go b/controllers/multi_cluster_app_dispatcher.go index b84ca6aaf..f49d9049e 100644 --- a/controllers/multi_cluster_app_dispatcher.go +++ b/controllers/multi_cluster_app_dispatcher.go @@ -1,7 +1,23 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package controllers import ( - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/v1alpha1" + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" ) var multiClusterAppDispatcherTemplates = []string{ diff --git a/controllers/suite_test.go b/controllers/suite_test.go index 67d6fa432..074422785 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -22,28 +22,30 @@ import ( "testing" "time" - "github.com/project-codeflare/codeflare-operator/controllers/util" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + mf "github.com/manifestival/manifestival" "go.uber.org/zap/zapcore" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/types" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - mf "github.com/manifestival/manifestival" - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/v1alpha1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/kubernetes/scheme" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" - //+kubebuilder:scaffold:imports + + "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" + "github.com/project-codeflare/codeflare-operator/controllers/util" + // +kubebuilder:scaffold:imports ) // These tests use Ginkgo (BDD-style Go testing framework). Refer to @@ -93,8 +95,8 @@ var _ = BeforeSuite(func() { // Register API objects utilruntime.Must(clientgoscheme.AddToScheme(scheme.Scheme)) - utilruntime.Must(codeflarev1alpha1.AddToScheme(scheme.Scheme)) - //+kubebuilder:scaffold:scheme + utilruntime.Must(v1alpha1.AddToScheme(scheme.Scheme)) + // +kubebuilder:scaffold:scheme // Initialize Kubernetes client k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) @@ -147,8 +149,8 @@ var _ = AfterSuite(func() { // Cleanup resources to not contaminate between tests var _ = AfterEach(func() { inNamespace := client.InNamespace(workingNamespace) - Expect(k8sClient.DeleteAllOf(context.TODO(), &codeflarev1alpha1.MCAD{}, inNamespace)).ToNot(HaveOccurred()) - Expect(k8sClient.DeleteAllOf(context.TODO(), &codeflarev1alpha1.InstaScale{}, inNamespace)).ToNot(HaveOccurred()) + Expect(k8sClient.DeleteAllOf(context.TODO(), &v1alpha1.MCAD{}, inNamespace)).ToNot(HaveOccurred()) + Expect(k8sClient.DeleteAllOf(context.TODO(), &v1alpha1.InstaScale{}, inNamespace)).ToNot(HaveOccurred()) }) @@ -181,7 +183,7 @@ func compareConfigMaps(path string, opts mf.Option) { Expect(util.ConfigMapsAreEqual(*expectedConfigMap, *actualConfigMap)).Should(BeTrue()) } -//func compareRoleBindings(path string, opts mf.Option) { +// func compareRoleBindings(path string, opts mf.Option) { // expectedRB := &k8srbacv1.RoleBinding{} // Expect(convertToStructuredResource(path, expectedRB, opts)).NotTo(HaveOccurred()) // expectedRB.Subjects[0].Namespace = workingNamespace @@ -193,7 +195,7 @@ func compareConfigMaps(path string, opts mf.Option) { // }, timeout, interval).ShouldNot(HaveOccurred()) // // Expect(util.RoleBindingsAreEqual(*expectedRB, *actualRB)).Should(BeTrue()) -//} +// } func compareServiceAccounts(path string, opts mf.Option) { expectedSA := &corev1.ServiceAccount{} diff --git a/go.mod b/go.mod index 5531c4777..b898acdef 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( k8s.io/apimachinery v0.26.3 k8s.io/client-go v0.26.3 sigs.k8s.io/controller-runtime v0.14.6 + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 ) require ( @@ -71,6 +72,5 @@ require ( k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 // indirect sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/main.go b/main.go index ff7e6886a..5358e04bd 100644 --- a/main.go +++ b/main.go @@ -30,13 +30,14 @@ import ( "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/v1alpha1" + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" "github.com/project-codeflare/codeflare-operator/controllers" - //+kubebuilder:scaffold:imports + // +kubebuilder:scaffold:imports ) var ( @@ -49,7 +50,7 @@ func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(codeflarev1alpha1.AddToScheme(scheme)) - //+kubebuilder:scaffold:scheme + // +kubebuilder:scaffold:scheme } func main() { @@ -101,7 +102,7 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "InstaScale") os.Exit(1) } - //+kubebuilder:scaffold:builder + // +kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") From 02e14b535b4f7172b0b809bcae4025008a1a968b Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 14 Jun 2023 18:33:06 +0200 Subject: [PATCH 022/100] Add MCAD version variable --- Makefile | 5 +++++ config/crd/mcad/kustomization.yaml | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2fd51060f..330d3647e 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,9 @@ VERSION ?= 0.0.0-dev # INSTASCALE_VERSION defines the default version of the InstaScale controller INSTASCALE_VERSION ?= v0.0.4 +# MCAD_VERSION defines the default version of the MCAD controller +MCAD_VERSION ?= v1.31.0 + # CHANNELS define the bundle channels used in the bundle. # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") # To re-generate a bundle for other specific channels without changing the standard setup, you can: @@ -200,6 +203,7 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified .PHONY: deploy deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) fn run config/crd/mcad --image gcr.io/kpt-fn/apply-setters:v0.2.0 -- MCAD_VERSION=$(MCAD_VERSION) cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} $(KUSTOMIZE) build config/default | kubectl apply -f - git restore config/* @@ -297,6 +301,7 @@ validate-bundle: install-operator-sdk .PHONY: bundle bundle: defaults manifests kustomize install-operator-sdk ## Generate bundle manifests and metadata, then validate generated files. $(OPERATOR_SDK) generate kustomize manifests -q + $(KUSTOMIZE) fn run config/crd/mcad --image gcr.io/kpt-fn/apply-setters:v0.2.0 -- MCAD_VERSION=$(MCAD_VERSION) cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) cd config/manifests && $(KUSTOMIZE) edit add patch --patch '[{"op":"add", "path":"/metadata/annotations/containerImage", "value": "$(IMG)" }]' --kind ClusterServiceVersion cd config/manifests && $(KUSTOMIZE) edit add patch --patch '[{"op":"add", "path":"/spec/replaces", "value": "codeflare-operator.v$(PREVIOUS_VERSION)" }]' --kind ClusterServiceVersion diff --git a/config/crd/mcad/kustomization.yaml b/config/crd/mcad/kustomization.yaml index 1474b25fc..14292a9a6 100644 --- a/config/crd/mcad/kustomization.yaml +++ b/config/crd/mcad/kustomization.yaml @@ -1,2 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization resources: - - github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=release-v1.31.0 +- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=release-v0.0.0 # kpt-set: github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=release-${MCAD_VERSION} From 73db60fcf09bc7d7cde8e0610ab7a797e578d915 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 16 Jun 2023 13:40:08 +0200 Subject: [PATCH 023/100] Extended support for referencing MCAD CRD resources --- Makefile | 8 ++++++-- config/crd/mcad/kustomization.yaml | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 330d3647e..6a47b4c9c 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,10 @@ INSTASCALE_VERSION ?= v0.0.4 # MCAD_VERSION defines the default version of the MCAD controller MCAD_VERSION ?= v1.31.0 +# MCAD_REF, MCAD_REPO and MCAD_CRD define the reference to MCAD CRD resources +MCAD_REF ?= release-${MCAD_VERSION} +MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher +MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_REF} # CHANNELS define the bundle channels used in the bundle. # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") @@ -203,7 +207,7 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified .PHONY: deploy deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) fn run config/crd/mcad --image gcr.io/kpt-fn/apply-setters:v0.2.0 -- MCAD_VERSION=$(MCAD_VERSION) + $(KUSTOMIZE) fn run config/crd/mcad --image gcr.io/kpt-fn/apply-setters:v0.2.0 -- MCAD_CRD=$(MCAD_CRD) cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} $(KUSTOMIZE) build config/default | kubectl apply -f - git restore config/* @@ -301,7 +305,7 @@ validate-bundle: install-operator-sdk .PHONY: bundle bundle: defaults manifests kustomize install-operator-sdk ## Generate bundle manifests and metadata, then validate generated files. $(OPERATOR_SDK) generate kustomize manifests -q - $(KUSTOMIZE) fn run config/crd/mcad --image gcr.io/kpt-fn/apply-setters:v0.2.0 -- MCAD_VERSION=$(MCAD_VERSION) + $(KUSTOMIZE) fn run config/crd/mcad --image gcr.io/kpt-fn/apply-setters:v0.2.0 -- MCAD_CRD=$(MCAD_CRD) cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) cd config/manifests && $(KUSTOMIZE) edit add patch --patch '[{"op":"add", "path":"/metadata/annotations/containerImage", "value": "$(IMG)" }]' --kind ClusterServiceVersion cd config/manifests && $(KUSTOMIZE) edit add patch --patch '[{"op":"add", "path":"/spec/replaces", "value": "codeflare-operator.v$(PREVIOUS_VERSION)" }]' --kind ClusterServiceVersion diff --git a/config/crd/mcad/kustomization.yaml b/config/crd/mcad/kustomization.yaml index 14292a9a6..a433f8335 100644 --- a/config/crd/mcad/kustomization.yaml +++ b/config/crd/mcad/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=release-v0.0.0 # kpt-set: github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=release-${MCAD_VERSION} +- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=release-v0.0.0 # kpt-set: ${MCAD_CRD} From 5e76774038b3ef65717144075f33e187ef1bcce6 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Thu, 15 Jun 2023 12:19:29 +0100 Subject: [PATCH 024/100] updating rbacs for instascale --- config/internal/instascale/clusterrole.yaml.tmpl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/config/internal/instascale/clusterrole.yaml.tmpl b/config/internal/instascale/clusterrole.yaml.tmpl index 13eb1461d..2fe66ef42 100644 --- a/config/internal/instascale/clusterrole.yaml.tmpl +++ b/config/internal/instascale/clusterrole.yaml.tmpl @@ -21,6 +21,13 @@ rules: - delete - patch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - apiGroups: - apps resources: From c2b68fab4966736637a646d9ecfa1ab2e56e1e73 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Thu, 15 Jun 2023 14:15:48 +0100 Subject: [PATCH 025/100] addressing feedback, specifying required secret --- config/internal/instascale/clusterrole.yaml.tmpl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/internal/instascale/clusterrole.yaml.tmpl b/config/internal/instascale/clusterrole.yaml.tmpl index 2fe66ef42..333fbf24a 100644 --- a/config/internal/instascale/clusterrole.yaml.tmpl +++ b/config/internal/instascale/clusterrole.yaml.tmpl @@ -23,6 +23,8 @@ rules: - apiGroups: - "" + resourceNames: + - instascale-ocm-secret resources: - secrets verbs: From 892d65dc92a5c79c0b9b97ee658930250dbf6b04 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Mon, 19 Jun 2023 15:41:13 +0100 Subject: [PATCH 026/100] adding clusterversions --- config/internal/instascale/clusterrole.yaml.tmpl | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/config/internal/instascale/clusterrole.yaml.tmpl b/config/internal/instascale/clusterrole.yaml.tmpl index 333fbf24a..8f27f57f0 100644 --- a/config/internal/instascale/clusterrole.yaml.tmpl +++ b/config/internal/instascale/clusterrole.yaml.tmpl @@ -24,11 +24,19 @@ rules: - apiGroups: - "" resourceNames: - - instascale-ocm-secret + - instascale-ocm-secret resources: - - secrets + - secrets verbs: - - get + - get + +- apiGroups: + - config.openshift.io + resources: + - clusterversions + verbs: + - get + - list - apiGroups: - apps From 29bd18e992dc605b54a191835c2f5c00f6a0d027 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Mon, 19 Jun 2023 15:55:09 +0100 Subject: [PATCH 027/100] updating test data --- .../case_1/clusterrole.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/controllers/testdata/instascale_test_results/case_1/clusterrole.yaml b/controllers/testdata/instascale_test_results/case_1/clusterrole.yaml index 11a22d454..c4053cdb1 100644 --- a/controllers/testdata/instascale_test_results/case_1/clusterrole.yaml +++ b/controllers/testdata/instascale_test_results/case_1/clusterrole.yaml @@ -16,6 +16,24 @@ rules: resources: - nodes - configmaps + + - verbs: + - get + apiGroups: + - '' + resourceNames: + - instascale-ocm-secret + resources: + - secrets + + - verbs: + - get + - list + apiGroups: + - config.openshift.io + resources: + - clusterversions + - verbs: - list - watch From 9a7746b085bcf378cc574dd1040986a757fe1b94 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 1 Jun 2023 09:33:40 +0200 Subject: [PATCH 028/100] Update README.md with release steps --- README.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/README.md b/README.md index 19ca46cff..e64478317 100644 --- a/README.md +++ b/README.md @@ -10,3 +10,36 @@ CodeFlare Stack Compatibility Matrix | CodeFlare-SDK | v0.4.4 | | InstaScale | v0.0.4 | | KubeRay | v0.5.0 | + +## Release process + +Prerequisite: +- Build and release [MCAD](https://github.com/project-codeflare/multi-cluster-app-dispatcher) +- Build and release [InstaScale](https://github.com/project-codeflare/instascale) +- Build and release [CodeFlare-SDK](https://github.com/project-codeflare/codeflare-sdk) + +Release steps: +1. Invoke [tag-and-build.yml](https://github.com/project-codeflare/codeflare-operator/actions/workflows/tag-and-build.yml) GitHub action, this action will create a repository tag, build and push operator image. + +2. Check result of [tag-and-build.yml](https://github.com/project-codeflare/codeflare-operator/actions/workflows/tag-and-build.yml) GitHub action, it should pass. + +3. Update CodeFlare Stack Compatibility Matrix in operator README. + +4. Update InstaScale and MCAD versions: +- in [Makefile](https://github.com/project-codeflare/codeflare-operator/blob/02e14b535b4f7172b0b809bcae4025008a1a968b/Makefile#L12-L16). +- in [mcad/deployment.yaml.tmpl](https://github.com/project-codeflare/codeflare-operator/blob/main/config/internal/mcad/deployment.yaml.tmpl#L28) +- in [controllers/defaults.go](https://github.com/project-codeflare/codeflare-operator/blob/main/controllers/defaults.go) by running `make defaults` +- in [controllers/testdata/instascale_test_results/case_1/deployment.yaml](https://github.com/project-codeflare/codeflare-operator/blob/main/controllers/testdata/instascale_test_results/case_1/deployment.yaml) and [controllers/testdata/instascale_test_results/case_2/deployment.yaml](https://github.com/project-codeflare/codeflare-operator/blob/main/controllers/testdata/instascale_test_results/case_2/deployment.yaml) + +5. Create a release in CodeFlare operator repository, release notes should include new support matrix. + +6. Open a pull request to OpenShift community operators repository with latest bundle using make command, check that the created PR has proper content. +``` +make openshift-community-operator-release +``` + +7. Once merged, update component stable tags to point at the latest image release. + +8. Announce the new release in slack and mail lists, if any. + +9. Update the Distributed Workloads component in ODH (also copy/update the compatibility matrix). From cd0cd5430cffa2e5312eb76249e271429dd77d6a Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 23 Jun 2023 15:28:16 +0200 Subject: [PATCH 029/100] feat(api): Support custom MCAD container image --- Makefile | 4 ++ api/codeflare/v1alpha1/mcad_types.go | 11 +++++ .../bases/codeflare.codeflare.dev_mcads.yaml | 11 +++++ config/internal/mcad/deployment.yaml.tmpl | 4 +- controllers/defaults.go | 1 + controllers/mcad_controller.go | 8 +--- controllers/mcad_controller_test.go | 10 +++++ controllers/mcad_params.go | 13 +++--- .../testdata/mcad_test_cases/case_3.yaml | 6 +++ .../mcad_test_results/case_3/deployment.yaml | 45 +++++++++++++++++++ 10 files changed, 98 insertions(+), 15 deletions(-) create mode 100644 controllers/testdata/mcad_test_cases/case_3.yaml create mode 100644 controllers/testdata/mcad_test_results/case_3/deployment.yaml diff --git a/Makefile b/Makefile index 6a47b4c9c..163bfbfd9 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,9 @@ IMAGE_ORG_BASE ?= quay.io/project-codeflare # codeflare.dev/codeflare-operator-bundle:$VERSION and codeflare.dev/codeflare-operator-catalog:$VERSION. IMAGE_TAG_BASE ?= $(IMAGE_ORG_BASE)/codeflare-operator +# MCAD_IMAGE defines the default container image for the MCAD controller +MCAD_IMAGE ?= $(IMAGE_ORG_BASE)/mcad-controller:$(MCAD_REF) + # INSTASCALE_IMAGE defines the default container image for the InstaScale controller INSTASCALE_IMAGE ?= $(IMAGE_ORG_BASE)/instascale-controller:$(INSTASCALE_VERSION) @@ -117,6 +120,7 @@ defaults: @echo "// ***********************" >> $(DEFAULTS_FILE) @echo "" >> $(DEFAULTS_FILE) @echo "const (" >> $(DEFAULTS_FILE) + @echo " MCADImage = \"$(MCAD_IMAGE)\"" >> $(DEFAULTS_FILE) @echo " InstaScaleImage = \"$(INSTASCALE_IMAGE)\"" >> $(DEFAULTS_FILE) @echo "" >> $(DEFAULTS_FILE) @echo ")" >> $(DEFAULTS_FILE) diff --git a/api/codeflare/v1alpha1/mcad_types.go b/api/codeflare/v1alpha1/mcad_types.go index 217791a21..dc22291f1 100644 --- a/api/codeflare/v1alpha1/mcad_types.go +++ b/api/codeflare/v1alpha1/mcad_types.go @@ -53,6 +53,17 @@ type MCADSpec struct { // ControllerResources defines the cpu and memory resource requirements for the MCAD Controller // +kubebuilder:default={} ControllerResources v1.ResourceRequirements `json:"controllerResources,omitempty" protobuf:"bytes,8,opt"` + + // The container image for the MCAD controller deployment. + // If specified, the provided container image must be compatible with the running CodeFlare operator. + // Using an incompatible, or unrelated container image, will result in an undefined behavior. + // A CodeFlare operator upgrade will not upgrade the MCAD controller, that'll keep running this + // specified container image. + // If not specified, the latest version compatible with the running CodeFlare operator is used. + // A CodeFlare operator upgrade may upgrade the MCAD controller to a newer container image. + // + // +optional + ControllerImage string `json:"controllerImage,omitempty"` } // MCADStatus defines the observed state of MCAD diff --git a/config/crd/bases/codeflare.codeflare.dev_mcads.yaml b/config/crd/bases/codeflare.codeflare.dev_mcads.yaml index fdb418346..8172df332 100644 --- a/config/crd/bases/codeflare.codeflare.dev_mcads.yaml +++ b/config/crd/bases/codeflare.codeflare.dev_mcads.yaml @@ -39,6 +39,17 @@ spec: description: AgentConfigs determine paths to agent config file:deploymentName separated by commas(,). type: string + controllerImage: + description: The container image for the MCAD controller deployment. + If specified, the provided container image must be compatible with + the running CodeFlare operator. Using an incompatible, or unrelated + container image, will result in an undefined behavior. A CodeFlare + operator upgrade will not upgrade the MCAD controller, that'll keep + running this specified container image. If not specified, the latest + version compatible with the running CodeFlare operator is used. + A CodeFlare operator upgrade may upgrade the MCAD controller to + a newer container image. + type: string controllerResources: description: ControllerResources defines the cpu and memory resource requirements for the MCAD Controller diff --git a/config/internal/mcad/deployment.yaml.tmpl b/config/internal/mcad/deployment.yaml.tmpl index 7f6ebfa8d..810b6c138 100644 --- a/config/internal/mcad/deployment.yaml.tmpl +++ b/config/internal/mcad/deployment.yaml.tmpl @@ -19,13 +19,13 @@ spec: spec: containers: - name: mcad-controller - args: [ "--v", "4", "--logtostderr"] + args: ["--v", "4", "--logtostderr"] command: - mcad-controller envFrom: - configMapRef: name: mcad-{{.Name}}-config - image: 'quay.io/project-codeflare/mcad-controller:release-v1.31.0' + image: {{.ControllerImage}} imagePullPolicy: Always ports: - name: https diff --git a/controllers/defaults.go b/controllers/defaults.go index 3f3fe49f0..d3cac3d07 100644 --- a/controllers/defaults.go +++ b/controllers/defaults.go @@ -5,5 +5,6 @@ package controllers // *********************** const ( + MCADImage = "quay.io/project-codeflare/mcad-controller:release-v1.31.0" InstaScaleImage = "quay.io/project-codeflare/instascale-controller:v0.0.4" ) diff --git a/controllers/mcad_controller.go b/controllers/mcad_controller.go index 1dfe8b9b1..f21023cdb 100644 --- a/controllers/mcad_controller.go +++ b/controllers/mcad_controller.go @@ -151,11 +151,7 @@ func (r *MCADReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. mcadCustomResource.APIVersion, mcadCustomResource.Kind = gvk.Version, gvk.Kind } - err = params.ExtractParams(mcadCustomResource) - if err != nil { - log.Error(err, "Unable to parse MCAD custom resource") - return ctrl.Result{}, err - } + params.ExtractParams(mcadCustomResource) if mcadCustomResource.ObjectMeta.DeletionTimestamp.IsZero() { if !controllerutil.ContainsFinalizer(mcadCustomResource, finalizerName) { @@ -189,7 +185,7 @@ func (r *MCADReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. if err != nil { return ctrl.Result{}, err } - err = r.Client.Status().Update(context.Background(), mcadCustomResource) + err = r.Client.Status().Update(ctx, mcadCustomResource) if err != nil { return ctrl.Result{}, err } diff --git a/controllers/mcad_controller_test.go b/controllers/mcad_controller_test.go index ff1189c4c..35e892c35 100644 --- a/controllers/mcad_controller_test.go +++ b/controllers/mcad_controller_test.go @@ -21,6 +21,8 @@ const ( mcadConfigMap2 = "./testdata/mcad_test_results/case_2/configmap.yaml" mcadService2 = "./testdata/mcad_test_results/case_2/service.yaml" mcadServiceAccount2 = "./testdata/mcad_test_results/case_2/serviceaccount.yaml" + mcadCRCase3 = "./testdata/mcad_test_cases/case_3.yaml" + mcadDeployment3 = "./testdata/mcad_test_results/case_3/deployment.yaml" ) func deployMCAD(ctx context.Context, path string, opts mf.Option) { @@ -54,4 +56,12 @@ var _ = Describe("The MCAD Controller", func() { compareServices(mcadService2, opts) }) }) + + Context("In a namespace, when a MCAD resource with a custom image is deployed", func() { + + It("It should create a deployment", func() { + deployMCAD(ctx, mcadCRCase3, opts) + compareDeployments(mcadDeployment3, opts) + }) + }) }) diff --git a/controllers/mcad_params.go b/controllers/mcad_params.go index cedbdecc7..5992bede4 100644 --- a/controllers/mcad_params.go +++ b/controllers/mcad_params.go @@ -33,16 +33,17 @@ type MCADParams struct { QuotaRestURL string PodCreationTimeout int ControllerResources ControllerResources + ControllerImage string } -// type ControllerResources struct { -// v1.ResourceRequirements -// } - // ExtractParams is currently a straight-up copy. We can add in more complex validation at a later date -func (p *MCADParams) ExtractParams(mcad *mcadv1alpha1.MCAD) error { +func (p *MCADParams) ExtractParams(mcad *mcadv1alpha1.MCAD) { p.Name = mcad.Name p.Namespace = mcad.Namespace + p.ControllerImage = mcad.Spec.ControllerImage + if p.ControllerImage == "" { + p.ControllerImage = MCADImage + } p.Owner = mcad p.EnableMonitoring = mcad.Spec.EnableMonitoring p.MultiCluster = mcad.Spec.MultiCluster @@ -52,6 +53,4 @@ func (p *MCADParams) ExtractParams(mcad *mcadv1alpha1.MCAD) error { p.QuotaRestURL = mcad.Spec.QuotaRestURL p.PodCreationTimeout = mcad.Spec.PodCreationTimeout p.ControllerResources = ControllerResources{mcad.Spec.ControllerResources} - - return nil } diff --git a/controllers/testdata/mcad_test_cases/case_3.yaml b/controllers/testdata/mcad_test_cases/case_3.yaml new file mode 100644 index 000000000..c048e2e88 --- /dev/null +++ b/controllers/testdata/mcad_test_cases/case_3.yaml @@ -0,0 +1,6 @@ +apiVersion: codeflare.codeflare.dev/v1alpha1 +kind: MCAD +metadata: + name: custom-image +spec: + controllerImage: quay.io/project-codeflare/mcad-controller:custom diff --git a/controllers/testdata/mcad_test_results/case_3/deployment.yaml b/controllers/testdata/mcad_test_results/case_3/deployment.yaml new file mode 100644 index 000000000..5efa0df2a --- /dev/null +++ b/controllers/testdata/mcad_test_results/case_3/deployment.yaml @@ -0,0 +1,45 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: mcad-controller-custom-image + namespace: default + labels: + app: mcad-custom-image + component: multi-cluster-application-dispatcher +spec: + replicas: 1 + selector: + matchLabels: + app: mcad-custom-image + template: + metadata: + labels: + app: mcad-custom-image + component: multi-cluster-application-dispatcher + spec: + containers: + - name: mcad-controller + args: ["--v", "4", "--logtostderr"] + command: + - mcad-controller + envFrom: + - configMapRef: + name: mcad-custom-image-config + image: quay.io/project-codeflare/mcad-controller:custom + imagePullPolicy: Always + ports: + - name: https + containerPort: 6443 + protocol: TCP + - name: http + containerPort: 8080 + protocol: TCP + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - name: temp-vol + mountPath: /tmp + serviceAccountName: mcad-controller-custom-image + volumes: + - name: temp-vol + emptyDir: {} From 504a1e9dcf7717af2c3de8c61e1e4d81c7546ab6 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Tue, 13 Jun 2023 09:25:13 +0200 Subject: [PATCH 030/100] Create GitHub action to automate CodeFlare operator release --- .github/workflows/tag-and-build.yml | 122 +++++++++++++++++++++++----- CONTRIBUTING.md | 2 +- Makefile | 37 +++++---- README.md | 24 ++---- 4 files changed, 131 insertions(+), 54 deletions(-) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 335c666c5..413ccf2d8 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -7,35 +7,69 @@ on: version: description: 'Tag to be used for operator image' required: true - default: '0.0.0-dev' + default: 'v0.0.0-dev' replaces: description: 'The previous semantic version that this tag replaces.' required: true - default: '0.0.0-dev' + default: 'v0.0.0-dev' + mcad-version: + description: 'Published version of multi-cluster-app-dispatcher' + required: true + default: 'v0.0.0-dev' + codeflare-sdk-version: + description: 'Published version of CodeFlare-SDK' + required: true + default: 'v0.0.0-dev' + instascale-version: + description: 'Published version of InstaScale' + required: true + default: 'v0.0.0-dev' + is-stable: + description: 'Select if the built image should be tagged as stable' + required: true + type: boolean + quay-organization: + description: 'Quay organization used to push the built images to' + required: true + default: 'project-codeflare' + community-operators-prod-fork-organization: + description: 'Owner of forked community-operators-prod repository used to push bundle files to' + required: true + default: 'project-codeflare' + community-operators-prod-organization: + description: 'Owner of target community-operators-prod repository used to open a PR against' + required: true + default: 'redhat-openshift-ecosystem' jobs: push: runs-on: ubuntu-latest + + # Permission required to create a release + permissions: + contents: write + steps: - uses: actions/checkout@v3 + - name: Verify that release doesn't exist yet + shell: bash {0} + run: | + gh release view ${{ github.event.inputs.version }} + status=$? + if [[ $status -eq 0 ]]; then + echo "Release ${{ github.event.inputs.version }} already exists." + exit 1 + fi + env: + GITHUB_TOKEN: ${{ github.TOKEN }} + - name: Activate cache uses: actions/cache@v3 with: path: /cache key: ${{ runner.os }}-cache-${{ hashFiles('**/go.sum', '.pre-commit-config.yaml') }} - - name: Create tag - uses: actions/github-script@v6 - with: - script: | - github.rest.git.createRef({ - owner: context.repo.owner, - repo: context.repo.repo, - ref: 'refs/tags/${{ github.event.inputs.version }}', - sha: context.sha - }) - - name: Install operator-sdk run: make install-operator-sdk @@ -46,18 +80,62 @@ jobs: password: ${{ secrets.QUAY_TOKEN }} registry: quay.io - - name: Image Build + - name: Image Build and Push run: | make build - make bundle - make image-build -e IMG=quay.io/project-codeflare/codeflare-operator:${SOURCE_TAG} - podman tag quay.io/project-codeflare/codeflare-operator:${SOURCE_TAG} quay.io/project-codeflare/codeflare-operator:latest + make image-build -e IMG=quay.io/${{ github.event.inputs.quay-organization }}/codeflare-operator:${{ github.event.inputs.version }} + make image-push -e IMG=quay.io/${{ github.event.inputs.quay-organization }}/codeflare-operator:${{ github.event.inputs.version }} + + - name: Image Push as stable tag + if: ${{ inputs.is-stable }} + run: | + podman tag quay.io/${{ github.event.inputs.quay-organization }}/codeflare-operator:${{ github.event.inputs.version }} quay.io/${{ github.event.inputs.quay-organization }}/codeflare-operator:stable + make image-push -e IMG=quay.io/${{ github.event.inputs.quay-organization }}/codeflare-operator:stable + + - name: Build bundle and create PR in OpenShift community operators repository + run: | + git config --global user.email "codeflare-ci@redhat.com" + git config --global user.name "CodeFlare CI" + make openshift-community-operator-release env: - SOURCE_TAG: ${{ github.event.inputs.version }} + VERSION: ${{ github.event.inputs.version }} + PREVIOUS_VERSION: ${{ github.event.inputs.replaces }} + INSTASCALE_VERSION: ${{ github.event.inputs.instascale-version }} + MCAD_VERSION: ${{ github.event.inputs.mcad-version }} + GH_TOKEN: ${{ secrets.GH_PAT }} + IMAGE_ORG_BASE: quay.io/${{ github.event.inputs.quay-organization }} + OPERATORS_REPO_FORK_ORG: ${{ github.event.inputs.community-operators-prod-fork-organization }} + OPERATORS_REPO_ORG: ${{ github.event.inputs.community-operators-prod-organization }} + + - name: Adjust Compatibility Matrix in readme + run: | + sed -i -E "s/(.*CodeFlare Operator.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.version }}\2/" README.md + sed -i -E "s/(.*Multi-Cluster App Dispatcher.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.mcad-version }}\2/" README.md + sed -i -E "s/(.*CodeFlare-SDK.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.codeflare-sdk-version }}\2/" README.md + sed -i -E "s/(.*InstaScale.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.instascale-version }}\2/" README.md + + - name: Adjust MCAD and InstaScale dependencies in the code + run: | + sed -i -E "s/(.*MCAD_VERSION \?= )v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.mcad-version }}\2/" Makefile + sed -i -E "s/(.*INSTASCALE_VERSION \?= )v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.instascale-version }}\2/" Makefile + sed -i -E "s/(.*instascale-controller:)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.instascale-version }}\2/" controllers/testdata/instascale_test_results/case_1/deployment.yaml + sed -i -E "s/(.*instascale-controller:)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.instascale-version }}\2/" controllers/testdata/instascale_test_results/case_2/deployment.yaml + + - name: Commit readme changes back to repository + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: Update dependency versions for release ${{ github.event.inputs.version }} + file_pattern: 'README.md controllers/defaults.go *.yaml *.tmpl Makefile' - - name: Image Push + - name: Creates a release in GitHub run: | - make image-push -e IMG=quay.io/project-codeflare/codeflare-operator:${SOURCE_TAG} - make image-push -e IMG=quay.io/project-codeflare/codeflare-operator:latest + gh release create ${{ github.event.inputs.version }} --target ${{ github.ref }} --generate-notes + # Edit notes to add there compatibility matrix + sed --null-data -E "s/(.*<\!-- Compatibility Matrix start -->)(.*)(<\!-- Compatibility Matrix end -->.*)/\2/" README.md > release-notes.md + echo "" >> release-notes.md + echo "$(gh release view --json body --jq .body)" >> release-notes.md + gh release edit ${{ github.event.inputs.version }} --notes-file release-notes.md + rm release-notes.md env: - SOURCE_TAG: ${{ github.event.inputs.version }} + GITHUB_TOKEN: ${{ github.TOKEN }} + shell: bash diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 85d98b387..f321c10d9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,7 +44,7 @@ For building and pushing a new version of the bundled operator image: - `make bundle-push -e IMAGE_TAG_BASE= VERSION= PREVIOUS_VERSION=` To create a new openshift-community-operator-release: - - `make openshift-community-operator-release -e IMAGE_TAG_BASE= VERSION= PREVIOUS_VERSION=` + - `make openshift-community-operator-release -e IMAGE_TAG_BASE= VERSION= PREVIOUS_VERSION= GH_TOKEN=` ## Testing The CodeFlare Operator currently has unit tests and pre-commit checks diff --git a/Makefile b/Makefile index 163bfbfd9..a6a5c1f17 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,15 @@ # VERSION defines the project version for the bundle. # Update this value when you upgrade the version of your project. # To re-generate a bundle for another specific version without changing the standard setup, you can: -# - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) -# - use environment variables to overwrite this value (e.g export VERSION=0.0.2) +# - use the VERSION as arg of the bundle target (e.g make bundle VERSION=v0.0.2) +# - use environment variables to overwrite this value (e.g export VERSION=v0.0.2) # best if we could detect this. If we cannot, we need to document it somewhere. # then we can add a patch in the `PHONY: bundle` +# BUNDLE_VERSION is declared as bundle versioning doesn't use semver -PREVIOUS_VERSION ?= 0.0.0-dev -VERSION ?= 0.0.0-dev +PREVIOUS_VERSION ?= v0.0.0-dev +VERSION ?= v0.0.0-dev +BUNDLE_VERSION ?= $(VERSION:v%=%) # INSTASCALE_VERSION defines the default version of the InstaScale controller INSTASCALE_VERSION ?= v0.0.4 @@ -19,6 +21,11 @@ MCAD_REF ?= release-${MCAD_VERSION} MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_REF} +# OPERATORS_REPO_ORG points to GitHub repository organization where bundle PR is opened against +# OPERATORS_REPO_FORK_ORG points to GitHub repository fork organization where bundle build is pushed to +OPERATORS_REPO_ORG ?= redhat-openshift-ecosystem +OPERATORS_REPO_FORK_ORG ?= project-codeflare + # CHANNELS define the bundle channels used in the bundle. # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") # To re-generate a bundle for other specific channels without changing the standard setup, you can: @@ -56,10 +63,10 @@ INSTASCALE_IMAGE ?= $(IMAGE_ORG_BASE)/instascale-controller:$(INSTASCALE_VERSION # BUNDLE_IMG defines the image:tag used for the bundle. # You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) -BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION) +BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:$(VERSION) # BUNDLE_GEN_FLAGS are the flags passed to the operator-sdk generate bundle command -BUNDLE_GEN_FLAGS ?= -q --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS) +BUNDLE_GEN_FLAGS ?= -q --overwrite --version $(BUNDLE_VERSION) $(BUNDLE_METADATA_OPTS) # USE_IMAGE_DIGESTS defines if images are resolved via tags or digests # You can enable this value if you would like to use SHA Based Digests @@ -70,7 +77,7 @@ ifeq ($(USE_IMAGE_DIGESTS), true) endif # Image URL to use all building/pushing image targets -IMG ?= ${IMAGE_TAG_BASE}:v${VERSION} +IMG ?= ${IMAGE_TAG_BASE}:${VERSION} # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. ENVTEST_K8S_VERSION = 1.24.2 @@ -312,7 +319,7 @@ bundle: defaults manifests kustomize install-operator-sdk ## Generate bundle man $(KUSTOMIZE) fn run config/crd/mcad --image gcr.io/kpt-fn/apply-setters:v0.2.0 -- MCAD_CRD=$(MCAD_CRD) cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) cd config/manifests && $(KUSTOMIZE) edit add patch --patch '[{"op":"add", "path":"/metadata/annotations/containerImage", "value": "$(IMG)" }]' --kind ClusterServiceVersion - cd config/manifests && $(KUSTOMIZE) edit add patch --patch '[{"op":"add", "path":"/spec/replaces", "value": "codeflare-operator.v$(PREVIOUS_VERSION)" }]' --kind ClusterServiceVersion + cd config/manifests && $(KUSTOMIZE) edit add patch --patch '[{"op":"add", "path":"/spec/replaces", "value": "codeflare-operator.$(PREVIOUS_VERSION)" }]' --kind ClusterServiceVersion $(KUSTOMIZE) build config/manifests | $(OPERATOR_SDK) generate bundle $(BUNDLE_GEN_FLAGS) $(MAKE) validate-bundle git restore config/* @@ -325,13 +332,13 @@ bundle-build: bundle ## Build the bundle image. bundle-push: ## Push the bundle image. $(MAKE) image-push IMG=$(BUNDLE_IMG) -.PHONY: openshift-community-operator-releases +.PHONY: openshift-community-operator-release openshift-community-operator-release: install-gh-cli bundle ## build bundle and create PR in OpenShift community operators repository - gh repo clone git@github.com:project-codeflare/community-operators-prod.git - cd community-operators-prod && git pull upstream main && git push origin main - cp -r bundle community-operators-prod/operators/codeflare-operator/$(VERSION) - cd community-operators-prod && git checkout -b codeflare-release-$(VERSION) && git add operators/codeflare-operator/$(VERSION)/* && git commit -s -m "add bundle manifests codeflare version $(VERSION)" && git push origin codeflare-release-$(VERSION) - gh pr create --repo redhat-openshift-ecosystem/community-operators-prod --title "CodeFlare $(VERSION)" --body "New release of codeflare operator" --head project-codeflare:codeflare-release-$(VERSION) --base main + git clone https://$(GH_TOKEN)@github.com/$(OPERATORS_REPO_FORK_ORG)/community-operators-prod.git + cd community-operators-prod && git remote add upstream https://github.com/$(OPERATORS_REPO_ORG)/community-operators-prod.git && git pull upstream main && git push origin main + cp -r bundle community-operators-prod/operators/codeflare-operator/$(BUNDLE_VERSION) + cd community-operators-prod && git checkout -b codeflare-release-$(BUNDLE_VERSION) && git add operators/codeflare-operator/$(BUNDLE_VERSION)/* && git commit -m "add bundle manifests codeflare version $(BUNDLE_VERSION)" && git push origin codeflare-release-$(BUNDLE_VERSION) + gh pr create --repo $(OPERATORS_REPO_FORK_ORG)/community-operators-prod --title "CodeFlare $(BUNDLE_VERSION)" --body "New release of codeflare operator" --head $(OPERATORS_REPO_ORG):codeflare-release-$(BUNDLE_VERSION) --base main rm -rf community-operators-prod .PHONY: opm @@ -356,7 +363,7 @@ endif BUNDLE_IMGS ?= $(BUNDLE_IMG) # The image tag given to the resulting catalog image (e.g. make catalog-build CATALOG_IMG=example.com/operator-catalog:v0.2.0). -CATALOG_IMG ?= $(IMAGE_TAG_BASE)-catalog:v$(VERSION) +CATALOG_IMG ?= $(IMAGE_TAG_BASE)-catalog:$(VERSION) # Set CATALOG_BASE_IMG to an existing catalog image tag to add $BUNDLE_IMGS to that image. ifneq ($(origin CATALOG_BASE_IMG), undefined) diff --git a/README.md b/README.md index e64478317..afc9dafae 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # codeflare-operator Operator for installation and lifecycle management of CodeFlare distributed workload stack, starting with MCAD and InstaScale + + CodeFlare Stack Compatibility Matrix | Component | Version | @@ -10,6 +12,7 @@ CodeFlare Stack Compatibility Matrix | CodeFlare-SDK | v0.4.4 | | InstaScale | v0.0.4 | | KubeRay | v0.5.0 | + ## Release process @@ -23,23 +26,12 @@ Release steps: 2. Check result of [tag-and-build.yml](https://github.com/project-codeflare/codeflare-operator/actions/workflows/tag-and-build.yml) GitHub action, it should pass. -3. Update CodeFlare Stack Compatibility Matrix in operator README. +3. Verify that compatibility matrix in [README](https://github.com/project-codeflare/codeflare-operator/blob/main/README.md) was properly updated. -4. Update InstaScale and MCAD versions: -- in [Makefile](https://github.com/project-codeflare/codeflare-operator/blob/02e14b535b4f7172b0b809bcae4025008a1a968b/Makefile#L12-L16). -- in [mcad/deployment.yaml.tmpl](https://github.com/project-codeflare/codeflare-operator/blob/main/config/internal/mcad/deployment.yaml.tmpl#L28) -- in [controllers/defaults.go](https://github.com/project-codeflare/codeflare-operator/blob/main/controllers/defaults.go) by running `make defaults` -- in [controllers/testdata/instascale_test_results/case_1/deployment.yaml](https://github.com/project-codeflare/codeflare-operator/blob/main/controllers/testdata/instascale_test_results/case_1/deployment.yaml) and [controllers/testdata/instascale_test_results/case_2/deployment.yaml](https://github.com/project-codeflare/codeflare-operator/blob/main/controllers/testdata/instascale_test_results/case_2/deployment.yaml) +4. Verify that opened pull request to [OpenShift community operators repository](https://github.com/redhat-openshift-ecosystem/community-operators-prod) has proper content. -5. Create a release in CodeFlare operator repository, release notes should include new support matrix. +5. Once PR is merged, update component stable tags to point at the latest image release. -6. Open a pull request to OpenShift community operators repository with latest bundle using make command, check that the created PR has proper content. -``` -make openshift-community-operator-release -``` +6. Announce the new release in slack and mail lists, if any. -7. Once merged, update component stable tags to point at the latest image release. - -8. Announce the new release in slack and mail lists, if any. - -9. Update the Distributed Workloads component in ODH (also copy/update the compatibility matrix). +7. Update the Distributed Workloads component in ODH (also copy/update the compatibility matrix). From 8e0c5a0cafdb54af6833f41d37c5d2c1fe2da5b7 Mon Sep 17 00:00:00 2001 From: Dimitri Saridakis Date: Tue, 27 Jun 2023 19:05:10 +0100 Subject: [PATCH 031/100] refactor: add liveness and readyness probes to instascale deployment (#150) --- config/internal/instascale/deployment.yaml.tmpl | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/config/internal/instascale/deployment.yaml.tmpl b/config/internal/instascale/deployment.yaml.tmpl index 4c704c80c..fe21ade62 100644 --- a/config/internal/instascale/deployment.yaml.tmpl +++ b/config/internal/instascale/deployment.yaml.tmpl @@ -21,4 +21,15 @@ spec: image: {{.ControllerImage}} name: instascale resources: {{.ControllerResources}} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + periodSeconds: 5 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + periodSeconds: 10 serviceAccountName: instascale-{{.Name}}-sa From 6531fc5cc811e886e0bd5b6e9ed61cf8926dd46c Mon Sep 17 00:00:00 2001 From: Dimitri Saridakis Date: Thu, 29 Jun 2023 12:02:59 +0100 Subject: [PATCH 032/100] perf: addition of liveness and readiness probes to mcad deployment template (#144) --- config/internal/mcad/deployment.yaml.tmpl | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/config/internal/mcad/deployment.yaml.tmpl b/config/internal/mcad/deployment.yaml.tmpl index 810b6c138..56cc691f4 100644 --- a/config/internal/mcad/deployment.yaml.tmpl +++ b/config/internal/mcad/deployment.yaml.tmpl @@ -40,6 +40,18 @@ spec: volumeMounts: - name: temp-vol mountPath: /tmp + livenessProbe: + httpGet: + path: /healthz + port: 8081 + timeoutSeconds: 5 + periodSeconds: 5 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + timeoutSeconds: 5 + periodSeconds: 5 serviceAccountName: mcad-controller-{{.Name}} volumes: - name: temp-vol From 88cd4762c339caf61ccb3fde25603e13aed56131 Mon Sep 17 00:00:00 2001 From: Dimitri Saridakis Date: Thu, 29 Jun 2023 16:18:49 +0100 Subject: [PATCH 033/100] fix: fixes the path for readiness probe --- config/internal/mcad/deployment.yaml.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/internal/mcad/deployment.yaml.tmpl b/config/internal/mcad/deployment.yaml.tmpl index 56cc691f4..4620c1dc7 100644 --- a/config/internal/mcad/deployment.yaml.tmpl +++ b/config/internal/mcad/deployment.yaml.tmpl @@ -48,7 +48,7 @@ spec: periodSeconds: 5 readinessProbe: httpGet: - path: /readyz + path: /healthz port: 8081 timeoutSeconds: 5 periodSeconds: 5 From f7a32268d9947aabb567db07fc5623c8477d6169 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 13 Jun 2023 12:01:42 +0200 Subject: [PATCH 034/100] Initial e2e tests --- .github/workflows/e2e_tests.yaml | 180 +++++++++++++++++++++++++++ .pre-commit-config.yaml | 1 - Makefile | 17 ++- go.mod | 2 + go.sum | 4 + test/e2e/kind.yaml | 32 +++++ test/e2e/mcad_test.go | 203 +++++++++++++++++++++++++++++++ test/support/client.go | 87 +++++++++++++ test/support/codeflare.go | 22 ++++ test/support/core.go | 17 +++ test/support/gomega.go | 28 +++++ test/support/mcad.go | 22 ++++ test/support/namespace.go | 56 +++++++++ test/support/support.go | 17 +++ test/support/test.go | 88 ++++++++++++++ test/support/utils.go | 5 + 16 files changed, 777 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/e2e_tests.yaml create mode 100644 test/e2e/kind.yaml create mode 100644 test/e2e/mcad_test.go create mode 100644 test/support/client.go create mode 100644 test/support/codeflare.go create mode 100644 test/support/core.go create mode 100644 test/support/gomega.go create mode 100644 test/support/mcad.go create mode 100644 test/support/namespace.go create mode 100644 test/support/support.go create mode 100644 test/support/test.go create mode 100644 test/support/utils.go diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml new file mode 100644 index 000000000..43e1c2ef5 --- /dev/null +++ b/.github/workflows/e2e_tests.yaml @@ -0,0 +1,180 @@ +name: e2e + +on: + pull_request: + branches: + - main + - 'release-*' + paths-ignore: + - 'docs/**' + - '**.adoc' + - '**.md' + - 'LICENSE' + push: + branches: + - main + - 'release-*' + paths-ignore: + - 'docs/**' + - '**.adoc' + - '**.md' + - 'LICENSE' + +concurrency: + group: ${{ github.head_ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + kubernetes: + + runs-on: ubuntu-20.04 + + steps: + - name: Cleanup + run: | + ls -lart + echo "Initial status:" + df -h + + echo "Cleaning up resources:" + sudo swapoff -a + sudo rm -f /swapfile + sudo apt clean + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + docker rmi $(docker image ls -aq) + + echo "Final status:" + df -h + + - name: Checkout code + uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Init directories + run: | + TEMP_DIR="$(pwd)/tmp" + mkdir -p "${TEMP_DIR}" + echo "TEMP_DIR=${TEMP_DIR}" >> $GITHUB_ENV + + mkdir -p "$(pwd)/bin" + echo "$(pwd)/bin" >> $GITHUB_PATH + + - name: Set Go + uses: actions/setup-go@v3 + with: + go-version: v1.18 + + - name: Container image registry + run: | + podman run -d -p 5000:5000 --name registry registry:2.8.1 + + export REGISTRY_ADDRESS=$(hostname -i):5000 + echo "REGISTRY_ADDRESS=${REGISTRY_ADDRESS}" >> $GITHUB_ENV + echo "Container image registry started at ${REGISTRY_ADDRESS}" + + KIND_CONFIG_FILE=${{ env.TEMP_DIR }}/kind.yaml + echo "KIND_CONFIG_FILE=${KIND_CONFIG_FILE}" >> $GITHUB_ENV + envsubst < ./test/e2e/kind.yaml > ${KIND_CONFIG_FILE} + + sudo --preserve-env=REGISTRY_ADDRESS sh -c 'cat > /etc/containers/registries.conf.d/local.conf < Date: Mon, 19 Jun 2023 17:05:36 +0200 Subject: [PATCH 035/100] test: Submit MNIST RayJob --- go.mod | 1 + go.sum | 2 + test/e2e/mcad_test.go | 80 +++++++++++++++++++++- test/e2e/mnist.py | 148 +++++++++++++++++++++++++++++++++++++++++ test/e2e/support.go | 6 ++ test/support/client.go | 13 ++++ test/support/ray.go | 24 +++++++ 7 files changed, 271 insertions(+), 3 deletions(-) create mode 100644 test/e2e/mnist.py create mode 100644 test/e2e/support.go create mode 100644 test/support/ray.go diff --git a/go.mod b/go.mod index 621853fb6..fff6657ef 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/onsi/gomega v1.27.6 github.com/project-codeflare/multi-cluster-app-dispatcher v1.31.0 github.com/ray-project/kuberay/ray-operator v0.0.0-20230614221720-085c29d40fa9 + github.com/rs/xid v1.5.0 go.uber.org/zap v1.24.0 k8s.io/api v0.26.3 k8s.io/apimachinery v0.26.3 diff --git a/go.sum b/go.sum index 8e877d1f6..c475743c8 100644 --- a/go.sum +++ b/go.sum @@ -465,6 +465,8 @@ github.com/ray-project/kuberay/ray-operator v0.0.0-20230614221720-085c29d40fa9 h github.com/ray-project/kuberay/ray-operator v0.0.0-20230614221720-085c29d40fa9/go.mod h1:2auArgwD9dXXJz1oc7SqQ4U/rHdpwnrBwG98kr8OWXA= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= diff --git a/test/e2e/mcad_test.go b/test/e2e/mcad_test.go index 5db8bf7c7..3c86f51a7 100644 --- a/test/e2e/mcad_test.go +++ b/test/e2e/mcad_test.go @@ -18,6 +18,7 @@ limitations under the License. package e2e import ( + "encoding/base64" "testing" . "github.com/onsi/gomega" @@ -26,10 +27,11 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" + "github.com/rs/xid" . "github.com/project-codeflare/codeflare-operator/test/support" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" ) func TestJobSubmissionInRayCluster(t *testing.T) { @@ -39,7 +41,29 @@ func TestJobSubmissionInRayCluster(t *testing.T) { // Create a namespace namespace := test.NewTestNamespace() + // Job script + mnist, err := scripts.ReadFile("mnist.py") + test.Expect(err).NotTo(HaveOccurred()) + + configMap := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.GroupName, + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "mnist", + Namespace: namespace.Name, + }, + BinaryData: map[string][]byte{ + "mnist.py": mnist, + }, + Immutable: Ptr(true), + } + configMap, err = test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), configMap, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + // RayCluster + clusterID := xid.New() rayCluster := &rayv1alpha1.RayCluster{ TypeMeta: metav1.TypeMeta{ APIVersion: rayv1alpha1.GroupVersion.String(), @@ -48,6 +72,9 @@ func TestJobSubmissionInRayCluster(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "raycluster-autoscaler", Namespace: namespace.Name, + Labels: map[string]string{ + RayJobDefaultClusterSelectorKey: clusterID.String(), + }, }, Spec: rayv1alpha1.RayClusterSpec{ RayVersion: "2.0.0", @@ -110,6 +137,24 @@ func TestJobSubmissionInRayCluster(t *testing.T) { corev1.ResourceMemory: resource.MustParse("1G"), }, }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "mnist", + MountPath: "/home/ray/jobs", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "mnist", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: configMap.Name, + }, + }, + }, }, }, }, @@ -195,9 +240,38 @@ func TestJobSubmissionInRayCluster(t *testing.T) { }, } - _, err := test.Client().MCAD().ArbV1().AppWrappers(namespace.Name).Create(aw) + _, err = test.Client().MCAD().ArbV1().AppWrappers(namespace.Name).Create(aw) test.Expect(err).NotTo(HaveOccurred()) test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) + + rayJob := &rayv1alpha1.RayJob{ + TypeMeta: metav1.TypeMeta{ + APIVersion: rayv1alpha1.GroupVersion.String(), + Kind: "RayJob", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "mnist", + Namespace: namespace.Name, + }, + Spec: rayv1alpha1.RayJobSpec{ + Entrypoint: "python /home/ray/jobs/mnist.py", + RuntimeEnv: base64.StdEncoding.EncodeToString([]byte(` +pytorch_lightning==1.5.10 +ray_lightning +torchmetrics==0.9.1 +torchvision==0.12.0 +`)), + ClusterSelector: map[string]string{ + RayJobDefaultClusterSelectorKey: clusterID.String(), + }, + ShutdownAfterJobFinishes: false, + }, + } + _, err = test.Client().Ray().RayV1alpha1().RayJobs(namespace.Name).Create(test.Ctx(), rayJob, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + + test.Eventually(RayJob(test, namespace, rayJob.Name), TestTimeoutLong). + Should(WithTransform(RayJobStatus, Equal(rayv1alpha1.JobStatusSucceeded))) } diff --git a/test/e2e/mnist.py b/test/e2e/mnist.py new file mode 100644 index 000000000..e60ec7c6e --- /dev/null +++ b/test/e2e/mnist.py @@ -0,0 +1,148 @@ +# In[] +import os + +import torch +from pytorch_lightning import LightningModule, Trainer +from pytorch_lightning.callbacks.progress import TQDMProgressBar +from pytorch_lightning.loggers import CSVLogger +from torch import nn +from torch.nn import functional as F +from torch.utils.data import DataLoader, random_split +from torchmetrics import Accuracy +from torchvision import transforms +from torchvision.datasets import MNIST + +PATH_DATASETS = os.environ.get("PATH_DATASETS", ".") +BATCH_SIZE = 256 if torch.cuda.is_available() else 64 +# %% + +print("prior to running the trainer") +print("MASTER_ADDR: is ", os.getenv("MASTER_ADDR")) +print("MASTER_PORT: is ", os.getenv("MASTER_PORT")) + + +class LitMNIST(LightningModule): + def __init__(self, data_dir=PATH_DATASETS, hidden_size=64, learning_rate=2e-4): + + super().__init__() + + # Set our init args as class attributes + self.data_dir = data_dir + self.hidden_size = hidden_size + self.learning_rate = learning_rate + + # Hardcode some dataset specific attributes + self.num_classes = 10 + self.dims = (1, 28, 28) + channels, width, height = self.dims + self.transform = transforms.Compose( + [ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,)), + ] + ) + + # Define PyTorch model + self.model = nn.Sequential( + nn.Flatten(), + nn.Linear(channels * width * height, hidden_size), + nn.ReLU(), + nn.Dropout(0.1), + nn.Linear(hidden_size, hidden_size), + nn.ReLU(), + nn.Dropout(0.1), + nn.Linear(hidden_size, self.num_classes), + ) + + self.val_accuracy = Accuracy() + self.test_accuracy = Accuracy() + + def forward(self, x): + x = self.model(x) + return F.log_softmax(x, dim=1) + + def training_step(self, batch, batch_idx): + x, y = batch + logits = self(x) + loss = F.nll_loss(logits, y) + return loss + + def validation_step(self, batch, batch_idx): + x, y = batch + logits = self(x) + loss = F.nll_loss(logits, y) + preds = torch.argmax(logits, dim=1) + self.val_accuracy.update(preds, y) + + # Calling self.log will surface up scalars for you in TensorBoard + self.log("val_loss", loss, prog_bar=True) + self.log("val_acc", self.val_accuracy, prog_bar=True) + + def test_step(self, batch, batch_idx): + x, y = batch + logits = self(x) + loss = F.nll_loss(logits, y) + preds = torch.argmax(logits, dim=1) + self.test_accuracy.update(preds, y) + + # Calling self.log will surface up scalars for you in TensorBoard + self.log("test_loss", loss, prog_bar=True) + self.log("test_acc", self.test_accuracy, prog_bar=True) + + def configure_optimizers(self): + optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate) + return optimizer + + #################### + # DATA RELATED HOOKS + #################### + + def prepare_data(self): + # download + print("Downloading MNIST dataset...") + MNIST(self.data_dir, train=True, download=True) + MNIST(self.data_dir, train=False, download=True) + + def setup(self, stage=None): + + # Assign train/val datasets for use in dataloaders + if stage == "fit" or stage is None: + mnist_full = MNIST(self.data_dir, train=True, transform=self.transform) + self.mnist_train, self.mnist_val = random_split(mnist_full, [55000, 5000]) + + # Assign test dataset for use in dataloader(s) + if stage == "test" or stage is None: + self.mnist_test = MNIST( + self.data_dir, train=False, transform=self.transform + ) + + def train_dataloader(self): + return DataLoader(self.mnist_train, batch_size=BATCH_SIZE) + + def val_dataloader(self): + return DataLoader(self.mnist_val, batch_size=BATCH_SIZE) + + def test_dataloader(self): + return DataLoader(self.mnist_test, batch_size=BATCH_SIZE) + + +# Init DataLoader from MNIST Dataset + +model = LitMNIST() + +print("GROUP: ", int(os.environ.get("GROUP_WORLD_SIZE", 1))) +print("LOCAL: ", int(os.environ.get("LOCAL_WORLD_SIZE", 1))) + +# Initialize a trainer +trainer = Trainer( + accelerator="auto", + # devices=1 if torch.cuda.is_available() else None, # limiting got iPython runs + max_epochs=5, + callbacks=[TQDMProgressBar(refresh_rate=20)], + num_nodes=int(os.environ.get("GROUP_WORLD_SIZE", 1)), + devices=int(os.environ.get("LOCAL_WORLD_SIZE", 1)), + strategy="ddp", +) + +# Train the model âš¡ +trainer.fit(model) diff --git a/test/e2e/support.go b/test/e2e/support.go new file mode 100644 index 000000000..27e9e54b4 --- /dev/null +++ b/test/e2e/support.go @@ -0,0 +1,6 @@ +package e2e + +import "embed" + +//go:embed *.py +var scripts embed.FS diff --git a/test/support/client.go b/test/support/client.go index 0026b2934..634df4f15 100644 --- a/test/support/client.go +++ b/test/support/client.go @@ -27,18 +27,21 @@ import ( codeflareclient "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" mcadclient "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/client/clientset/controller-versioned" + rayclient "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned" ) type Client interface { Core() kubernetes.Interface CodeFlare() codeflareclient.Interface MCAD() mcadclient.Interface + Ray() rayclient.Interface } type testClient struct { core kubernetes.Interface codeflare codeflareclient.Interface mcad mcadclient.Interface + ray rayclient.Interface } var _ Client = (*testClient)(nil) @@ -55,6 +58,10 @@ func (t *testClient) MCAD() mcadclient.Interface { return t.mcad } +func (t *testClient) Ray() rayclient.Interface { + return t.ray +} + func newTestClient() (Client, error) { cfg, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( clientcmd.NewDefaultClientConfigLoadingRules(), @@ -79,9 +86,15 @@ func newTestClient() (Client, error) { return nil, err } + rayClient, err := rayclient.NewForConfig(cfg) + if err != nil { + return nil, err + } + return &testClient{ core: kubeClient, codeflare: codeFlareClient, mcad: mcadClient, + ray: rayClient, }, nil } diff --git a/test/support/ray.go b/test/support/ray.go new file mode 100644 index 000000000..dfc3b2bb2 --- /dev/null +++ b/test/support/ray.go @@ -0,0 +1,24 @@ +package support + +import ( + "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" +) + +const RayJobDefaultClusterSelectorKey = "ray.io/cluster" + +func RayJob(t Test, namespace *corev1.Namespace, name string) func(g gomega.Gomega) *rayv1alpha1.RayJob { + return func(g gomega.Gomega) *rayv1alpha1.RayJob { + job, err := t.Client().Ray().RayV1alpha1().RayJobs(namespace.Name).Get(t.Ctx(), name, metav1.GetOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + return job + } +} + +func RayJobStatus(job *rayv1alpha1.RayJob) rayv1alpha1.JobStatus { + return job.Status.JobStatus +} From aea8000c4bd6c084128a93d106d0055dad8e57cc Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Mon, 19 Jun 2023 17:25:31 +0200 Subject: [PATCH 036/100] test: Fix RayCluster labels selector --- go.mod | 1 - go.sum | 2 -- test/e2e/mcad_test.go | 8 +------- 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/go.mod b/go.mod index fff6657ef..621853fb6 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,6 @@ require ( github.com/onsi/gomega v1.27.6 github.com/project-codeflare/multi-cluster-app-dispatcher v1.31.0 github.com/ray-project/kuberay/ray-operator v0.0.0-20230614221720-085c29d40fa9 - github.com/rs/xid v1.5.0 go.uber.org/zap v1.24.0 k8s.io/api v0.26.3 k8s.io/apimachinery v0.26.3 diff --git a/go.sum b/go.sum index c475743c8..8e877d1f6 100644 --- a/go.sum +++ b/go.sum @@ -465,8 +465,6 @@ github.com/ray-project/kuberay/ray-operator v0.0.0-20230614221720-085c29d40fa9 h github.com/ray-project/kuberay/ray-operator v0.0.0-20230614221720-085c29d40fa9/go.mod h1:2auArgwD9dXXJz1oc7SqQ4U/rHdpwnrBwG98kr8OWXA= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= -github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= diff --git a/test/e2e/mcad_test.go b/test/e2e/mcad_test.go index 3c86f51a7..4557484df 100644 --- a/test/e2e/mcad_test.go +++ b/test/e2e/mcad_test.go @@ -27,8 +27,6 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/rs/xid" - . "github.com/project-codeflare/codeflare-operator/test/support" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" @@ -63,7 +61,6 @@ func TestJobSubmissionInRayCluster(t *testing.T) { test.Expect(err).NotTo(HaveOccurred()) // RayCluster - clusterID := xid.New() rayCluster := &rayv1alpha1.RayCluster{ TypeMeta: metav1.TypeMeta{ APIVersion: rayv1alpha1.GroupVersion.String(), @@ -72,9 +69,6 @@ func TestJobSubmissionInRayCluster(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "raycluster-autoscaler", Namespace: namespace.Name, - Labels: map[string]string{ - RayJobDefaultClusterSelectorKey: clusterID.String(), - }, }, Spec: rayv1alpha1.RayClusterSpec{ RayVersion: "2.0.0", @@ -264,7 +258,7 @@ torchmetrics==0.9.1 torchvision==0.12.0 `)), ClusterSelector: map[string]string{ - RayJobDefaultClusterSelectorKey: clusterID.String(), + RayJobDefaultClusterSelectorKey: rayCluster.Name, }, ShutdownAfterJobFinishes: false, }, From 059af32eb3701fd1d875abd2dc3a6f3d2329f78e Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Mon, 19 Jun 2023 17:28:10 +0200 Subject: [PATCH 037/100] e2e: Print KubeRay operator logs --- .github/workflows/e2e_tests.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 43e1c2ef5..1f1a80e16 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -178,3 +178,9 @@ jobs: run: | echo "Printing MCAD controller logs" kubectl logs -n codeflare-system --tail -1 -l component=multi-cluster-application-dispatcher + + - name: Print KubeRay operator logs + if: always() && steps.deploy.outcome == 'success' + run: | + echo "Printing KubeRay operator logs" + kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay From fb4b7b4ce98aa62830dfdb336c1b4e89f1227ccb Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Mon, 19 Jun 2023 17:44:58 +0200 Subject: [PATCH 038/100] test: Fix RayJob runtime environment --- test/e2e/mcad_test.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/test/e2e/mcad_test.go b/test/e2e/mcad_test.go index 4557484df..930222a64 100644 --- a/test/e2e/mcad_test.go +++ b/test/e2e/mcad_test.go @@ -252,10 +252,16 @@ func TestJobSubmissionInRayCluster(t *testing.T) { Spec: rayv1alpha1.RayJobSpec{ Entrypoint: "python /home/ray/jobs/mnist.py", RuntimeEnv: base64.StdEncoding.EncodeToString([]byte(` -pytorch_lightning==1.5.10 -ray_lightning -torchmetrics==0.9.1 -torchvision==0.12.0 +{ + "pip": [ + "pytorch_lightning==1.5.10", + "ray_lightning", + "torchmetrics==0.9.1", + "torchvision==0.12.0" + ], + "env_vars": { + } +} `)), ClusterSelector: map[string]string{ RayJobDefaultClusterSelectorKey: rayCluster.Name, From 922dd7a142e093fca493310a394701a88f8c022e Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 20 Jun 2023 10:33:00 +0200 Subject: [PATCH 039/100] test: Print RayJob logs --- test/e2e/mcad_test.go | 7 ++++++- test/support/ray.go | 17 +++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/test/e2e/mcad_test.go b/test/e2e/mcad_test.go index 930222a64..263bbcb41 100644 --- a/test/e2e/mcad_test.go +++ b/test/e2e/mcad_test.go @@ -269,9 +269,14 @@ func TestJobSubmissionInRayCluster(t *testing.T) { ShutdownAfterJobFinishes: false, }, } - _, err = test.Client().Ray().RayV1alpha1().RayJobs(namespace.Name).Create(test.Ctx(), rayJob, metav1.CreateOptions{}) + rayJob, err = test.Client().Ray().RayV1alpha1().RayJobs(namespace.Name).Create(test.Ctx(), rayJob, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) test.Eventually(RayJob(test, namespace, rayJob.Name), TestTimeoutLong). Should(WithTransform(RayJobStatus, Equal(rayv1alpha1.JobStatusSucceeded))) + + rayJob, err = test.Client().Ray().RayV1alpha1().RayJobs(namespace.Name).Get(test.Ctx(), rayJob.Name, metav1.GetOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + + test.T().Log(GetRayJobLogs(test, rayJob)) } diff --git a/test/support/ray.go b/test/support/ray.go index dfc3b2bb2..3e1389edd 100644 --- a/test/support/ray.go +++ b/test/support/ray.go @@ -1,6 +1,8 @@ package support import ( + "encoding/json" + "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" @@ -22,3 +24,18 @@ func RayJob(t Test, namespace *corev1.Namespace, name string) func(g gomega.Gome func RayJobStatus(job *rayv1alpha1.RayJob) rayv1alpha1.JobStatus { return job.Status.JobStatus } + +func GetRayJobLogs(t Test, job *rayv1alpha1.RayJob) string { + t.T().Helper() + response := t.Client().Core().CoreV1().RESTClient(). + Get(). + AbsPath("/api/v1/namespaces", job.Namespace, "services", "http:"+job.Status.RayClusterName+"-head-svc:dashboard", "proxy", "api", "jobs", job.Status.JobId, "logs").Do(t.Ctx()) + t.Expect(response.Error()).NotTo(gomega.HaveOccurred()) + + body := map[string]string{} + bytes, _ := response.Raw() + t.Expect(json.Unmarshal(bytes, &body)).To(gomega.Succeed()) + t.Expect(body).To(gomega.HaveKey("logs")) + + return body["logs"] +} From d4a6cfefdec035160d68ccf7b48be879ee7ffa75 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 21 Jun 2023 14:44:26 +0200 Subject: [PATCH 040/100] test: Document how to run e2e tests locally --- README.md | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index afc9dafae..31f216405 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # codeflare-operator + Operator for installation and lifecycle management of CodeFlare distributed workload stack, starting with MCAD and InstaScale @@ -14,7 +15,36 @@ CodeFlare Stack Compatibility Matrix | KubeRay | v0.5.0 | -## Release process +## Development + +### Testing + +The e2e tests can be executed locally by running the following commands: + +1. Setup the test cluster: + + ```bash + # Create a KinD cluster + $ kind create cluster --image kindest/node:v1.25.8 + # Install the CRDs + $ make install + ``` + +2. Start the operator locally: + + ```bash + $ make run + ``` + +3. In a separate terminal, run the e2e suite: + + ```bash + $ make test-e2e + ``` + + Alternatively, You can run the e2e test(s) from your IDE / debugger. + +## Release Prerequisite: - Build and release [MCAD](https://github.com/project-codeflare/multi-cluster-app-dispatcher) From ff609f9be997e6675a340a8334ebfaecb1e205f7 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 21 Jun 2023 15:24:11 +0200 Subject: [PATCH 041/100] test: Polish MNIST RayJob test --- test/e2e/kind.yaml | 23 +++++++++---------- ...o => mnist_rayjob_mcad_raycluster_test.go} | 15 ++++++------ test/e2e/support.go | 16 +++++++++++++ test/support/client.go | 15 ++++++------ test/support/codeflare.go | 22 ------------------ test/support/core.go | 16 +++++++++++++ test/support/gomega.go | 16 +++++++++++++ test/support/mcad.go | 16 +++++++++++++ test/support/namespace.go | 15 ++++++------ test/support/ray.go | 16 +++++++++++++ test/support/support.go | 16 +++++++++++++ test/support/test.go | 16 +++++++++++++ test/support/utils.go | 16 +++++++++++++ 13 files changed, 160 insertions(+), 58 deletions(-) rename test/e2e/{mcad_test.go => mnist_rayjob_mcad_raycluster_test.go} (94%) delete mode 100644 test/support/codeflare.go diff --git a/test/e2e/kind.yaml b/test/e2e/kind.yaml index ebb1f7319..4546589b8 100644 --- a/test/e2e/kind.yaml +++ b/test/e2e/kind.yaml @@ -1,18 +1,17 @@ # --------------------------------------------------------------------------- -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at +# Copyright 2023. # -# http://www.apache.org/licenses/LICENSE-2.0 +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # --------------------------------------------------------------------------- kind: Cluster diff --git a/test/e2e/mcad_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go similarity index 94% rename from test/e2e/mcad_test.go rename to test/e2e/mnist_rayjob_mcad_raycluster_test.go index 263bbcb41..0fe82043c 100644 --- a/test/e2e/mcad_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -1,12 +1,11 @@ /* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at +Copyright 2023. - http://www.apache.org/licenses/LICENSE-2.0 +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -32,7 +31,7 @@ import ( rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" ) -func TestJobSubmissionInRayCluster(t *testing.T) { +func TestMNISTRayJobMCADRayCluster(t *testing.T) { test := With(t) test.T().Parallel() diff --git a/test/e2e/support.go b/test/e2e/support.go index 27e9e54b4..82f980ebd 100644 --- a/test/e2e/support.go +++ b/test/e2e/support.go @@ -1,3 +1,19 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package e2e import "embed" diff --git a/test/support/client.go b/test/support/client.go index 634df4f15..a8efa76d2 100644 --- a/test/support/client.go +++ b/test/support/client.go @@ -1,12 +1,11 @@ /* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/test/support/codeflare.go b/test/support/codeflare.go deleted file mode 100644 index 3c26caa29..000000000 --- a/test/support/codeflare.go +++ /dev/null @@ -1,22 +0,0 @@ -package support - -import ( - "github.com/onsi/gomega" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" -) - -func MCAD(t Test, namespace *corev1.Namespace, name string) func(g gomega.Gomega) *codeflarev1alpha1.MCAD { - return func(g gomega.Gomega) *codeflarev1alpha1.MCAD { - mcad, err := t.Client().CodeFlare().CodeflareV1alpha1().MCADs(namespace.Name).Get(t.Ctx(), name, metav1.GetOptions{}) - g.Expect(err).NotTo(gomega.HaveOccurred()) - return mcad - } -} - -func ReadyStatus(mcad *codeflarev1alpha1.MCAD) bool { - return mcad.Status.Ready -} diff --git a/test/support/core.go b/test/support/core.go index 21c3537ef..10fbd4b2f 100644 --- a/test/support/core.go +++ b/test/support/core.go @@ -1,3 +1,19 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package support import ( diff --git a/test/support/gomega.go b/test/support/gomega.go index 27bb72234..5631044a9 100644 --- a/test/support/gomega.go +++ b/test/support/gomega.go @@ -1,3 +1,19 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package support import ( diff --git a/test/support/mcad.go b/test/support/mcad.go index 0930fc385..4f268985f 100644 --- a/test/support/mcad.go +++ b/test/support/mcad.go @@ -1,3 +1,19 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package support import ( diff --git a/test/support/namespace.go b/test/support/namespace.go index e9f573992..145acbb4a 100644 --- a/test/support/namespace.go +++ b/test/support/namespace.go @@ -1,12 +1,11 @@ /* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/test/support/ray.go b/test/support/ray.go index 3e1389edd..369db80bf 100644 --- a/test/support/ray.go +++ b/test/support/ray.go @@ -1,3 +1,19 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package support import ( diff --git a/test/support/support.go b/test/support/support.go index 43a4951a8..184b01864 100644 --- a/test/support/support.go +++ b/test/support/support.go @@ -1,3 +1,19 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package support import ( diff --git a/test/support/test.go b/test/support/test.go index 5fcb6f6fc..5b3b271b7 100644 --- a/test/support/test.go +++ b/test/support/test.go @@ -1,3 +1,19 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package support import ( diff --git a/test/support/utils.go b/test/support/utils.go index 9ddf287d8..ed40309f2 100644 --- a/test/support/utils.go +++ b/test/support/utils.go @@ -1,3 +1,19 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package support func Ptr[T any](v T) *T { From 2361ef5216c36da1b1bc862c5705ac9c343748ed Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 22 Jun 2023 09:50:47 +0200 Subject: [PATCH 042/100] test: Add MNIST training with MCAD Job --- test/e2e/mnist_pytorch_mcad_job_test.go | 182 ++++++++++++++++++ test/e2e/mnist_rayjob_mcad_raycluster_test.go | 2 +- test/support/batch.go | 33 ++++ test/support/conditions.go | 53 +++++ 4 files changed, 269 insertions(+), 1 deletion(-) create mode 100644 test/e2e/mnist_pytorch_mcad_job_test.go create mode 100644 test/support/batch.go create mode 100644 test/support/conditions.go diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go new file mode 100644 index 000000000..242bef05b --- /dev/null +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -0,0 +1,182 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + . "github.com/onsi/gomega" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/project-codeflare/codeflare-operator/test/support" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" +) + +func TestMNISTPyTorchMCAD(t *testing.T) { + test := With(t) + test.T().Parallel() + + // Create a namespace + namespace := test.NewTestNamespace() + + // MNIST training script + mnist, err := scripts.ReadFile("mnist.py") + test.Expect(err).NotTo(HaveOccurred()) + + mnistScript := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "mnist", + Namespace: namespace.Name, + }, + BinaryData: map[string][]byte{ + "mnist.py": mnist, + }, + Immutable: Ptr(true), + } + mnistScript, err = test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), mnistScript, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + + // pip requirements + requirements := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "requirements", + Namespace: namespace.Name, + }, + BinaryData: map[string][]byte{ + "requirements.txt": []byte(` +pytorch_lightning==1.5.10 +torchmetrics==0.9.1 +torchvision==0.12.0 +`), + }, + Immutable: Ptr(true), + } + requirements, err = test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), requirements, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + + // Batch Job + job := &batchv1.Job{ + TypeMeta: metav1.TypeMeta{ + APIVersion: batchv1.SchemeGroupVersion.String(), + Kind: "Job", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "mnist", + Namespace: namespace.Name, + }, + Spec: batchv1.JobSpec{ + Completions: Ptr(int32(1)), + Parallelism: Ptr(int32(1)), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "job", + Image: "pytorch/pytorch:1.11.0-cuda11.3-cudnn8-runtime", + Command: []string{"/bin/sh", "-c", "pip install -r /test/runtime/requirements.txt && torchrun /test/job/mnist.py"}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "mnist", + MountPath: "/test/job", + }, + { + Name: "requirements", + MountPath: "/test/runtime", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "mnist", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: mnistScript.Name, + }, + }, + }, + }, + { + Name: "requirements", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: requirements.Name, + }, + }, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + }, + }, + }, + } + + // Create an AppWrapper resource + aw := &mcadv1beta1.AppWrapper{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mnist", + Namespace: namespace.Name, + }, + Spec: mcadv1beta1.AppWrapperSpec{ + AggrResources: mcadv1beta1.AppWrapperResourceList{ + GenericItems: []mcadv1beta1.AppWrapperGenericResource{ + { + DesiredAvailable: 1, + CustomPodResources: []mcadv1beta1.CustomPodResourceTemplate{ + { + Replicas: 1, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("250m"), + corev1.ResourceMemory: resource.MustParse("512Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("1G"), + }, + }, + }, + GenericTemplate: Raw(test, job), + }, + }, + }, + }, + } + + _, err = test.Client().MCAD().ArbV1().AppWrappers(namespace.Name).Create(aw) + test.Expect(err).NotTo(HaveOccurred()) + + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). + Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) + + test.Eventually(Job(test, namespace, job.Name), TestTimeoutLong). + Should(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) +} diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index 0fe82043c..0af97f5c7 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -44,7 +44,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { configMap := &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.GroupName, + APIVersion: corev1.SchemeGroupVersion.String(), Kind: "ConfigMap", }, ObjectMeta: metav1.ObjectMeta{ diff --git a/test/support/batch.go b/test/support/batch.go new file mode 100644 index 000000000..4aea06fd7 --- /dev/null +++ b/test/support/batch.go @@ -0,0 +1,33 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package support + +import ( + "github.com/onsi/gomega" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func Job(t Test, namespace *corev1.Namespace, name string) func(g gomega.Gomega) *batchv1.Job { + return func(g gomega.Gomega) *batchv1.Job { + job, err := t.Client().Core().BatchV1().Jobs(namespace.Name).Get(t.Ctx(), name, metav1.GetOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + return job + } +} diff --git a/test/support/conditions.go b/test/support/conditions.go new file mode 100644 index 000000000..e7c5097ad --- /dev/null +++ b/test/support/conditions.go @@ -0,0 +1,53 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package support + +import ( + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" +) + +type conditionType interface { + ~string +} + +func ConditionStatus[T conditionType](conditionType T) func(any) corev1.ConditionStatus { + return func(object any) corev1.ConditionStatus { + switch o := object.(type) { + + case *batchv1.Job: + if c := getJobCondition(o.Status.Conditions, batchv1.JobConditionType(conditionType)); c != nil { + return c.Status + } + + } + + return corev1.ConditionUnknown + } +} + +// TODO: to be replaced with a generic version once common struct fields of a type set can be used. +// See https://github.com/golang/go/issues/48522 +func getJobCondition(conditions []batchv1.JobCondition, conditionType batchv1.JobConditionType) *batchv1.JobCondition { + for _, c := range conditions { + if c.Type == conditionType { + return &c + } + } + return nil +} From 1285e8e8f5409bf70465de389e318ed53c48cbf8 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 22 Jun 2023 10:54:53 +0200 Subject: [PATCH 043/100] test: Print MNIST batch job logs --- test/e2e/mnist_pytorch_mcad_job_test.go | 13 +++++++++++++ test/support/batch.go | 5 +++++ test/support/core.go | 26 +++++++++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index 242bef05b..d305d9a6c 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -25,6 +25,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" . "github.com/project-codeflare/codeflare-operator/test/support" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" @@ -179,4 +180,16 @@ torchvision==0.12.0 test.Eventually(Job(test, namespace, job.Name), TestTimeoutLong). Should(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) + + // Refresh the job to get the generated pod selector + job = GetJob(test, namespace, job.Name) + + // Get the job Pod + pods := GetPods(test, namespace, metav1.ListOptions{ + LabelSelector: labels.FormatLabels(job.Spec.Selector.MatchLabels)}, + ) + test.Expect(pods).To(HaveLen(1)) + + // Print the job logs + test.T().Log(GetPodLogs(test, &pods[0], corev1.PodLogOptions{})) } diff --git a/test/support/batch.go b/test/support/batch.go index 4aea06fd7..c868d7387 100644 --- a/test/support/batch.go +++ b/test/support/batch.go @@ -31,3 +31,8 @@ func Job(t Test, namespace *corev1.Namespace, name string) func(g gomega.Gomega) return job } } + +func GetJob(t Test, namespace *corev1.Namespace, name string) *batchv1.Job { + t.T().Helper() + return Job(t, namespace, name)(t) +} diff --git a/test/support/core.go b/test/support/core.go index 10fbd4b2f..9da6d0b3f 100644 --- a/test/support/core.go +++ b/test/support/core.go @@ -18,8 +18,12 @@ package support import ( "encoding/json" + "io" "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -31,3 +35,25 @@ func Raw(t Test, obj runtime.Object) runtime.RawExtension { Raw: data, } } + +func GetPods(t Test, namespace *corev1.Namespace, options metav1.ListOptions) []corev1.Pod { + t.T().Helper() + pods, err := t.Client().Core().CoreV1().Pods(namespace.Name).List(t.Ctx(), options) + t.Expect(err).NotTo(gomega.HaveOccurred()) + return pods.Items +} + +func GetPodLogs(t Test, pod *corev1.Pod, options corev1.PodLogOptions) string { + t.T().Helper() + stream, err := t.Client().Core().CoreV1().Pods(pod.GetNamespace()).GetLogs(pod.GetName(), &options).Stream(t.Ctx()) + t.Expect(err).NotTo(gomega.HaveOccurred()) + + defer func() { + t.Expect(stream.Close()).To(gomega.Succeed()) + }() + + bytes, err := io.ReadAll(stream) + t.Expect(err).NotTo(gomega.HaveOccurred()) + + return string(bytes) +} From 6553bfd2e7d591517241cadb405db655bd05e91f Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 22 Jun 2023 11:07:21 +0200 Subject: [PATCH 044/100] test: Use RayCluster 'complete' configuration --- test/e2e/mnist_rayjob_mcad_raycluster_test.go | 45 +++++-------------- 1 file changed, 12 insertions(+), 33 deletions(-) diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index 0af97f5c7..8e8f97f8f 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -38,7 +38,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { // Create a namespace namespace := test.NewTestNamespace() - // Job script + // MNIST training script mnist, err := scripts.ReadFile("mnist.py") test.Expect(err).NotTo(HaveOccurred()) @@ -66,39 +66,21 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { Kind: "RayCluster", }, ObjectMeta: metav1.ObjectMeta{ - Name: "raycluster-autoscaler", + Name: "raycluster", Namespace: namespace.Name, }, Spec: rayv1alpha1.RayClusterSpec{ - RayVersion: "2.0.0", - EnableInTreeAutoscaling: Ptr(true), - AutoscalerOptions: &rayv1alpha1.AutoscalerOptions{ - UpscalingMode: Ptr[rayv1alpha1.UpscalingMode]("Default"), - IdleTimeoutSeconds: Ptr(int32(60)), - ImagePullPolicy: Ptr(corev1.PullAlways), - Resources: &corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("250m"), - corev1.ResourceMemory: resource.MustParse("512Mi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("250m"), - corev1.ResourceMemory: resource.MustParse("512Mi"), - }, - }, - }, + RayVersion: "2.0.0", HeadGroupSpec: rayv1alpha1.HeadGroupSpec{ RayStartParams: map[string]string{ "dashboard-host": "0.0.0.0", - "block": "true", }, Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ { - Name: "ray-head", - Image: "rayproject/ray:2.0.0", - ImagePullPolicy: corev1.PullAlways, + Name: "ray-head", + Image: "rayproject/ray:2.0.0", Ports: []corev1.ContainerPort{ { ContainerPort: 6379, @@ -155,13 +137,11 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { }, WorkerGroupSpecs: []rayv1alpha1.WorkerGroupSpec{ { - Replicas: Ptr(int32(1)), - MinReplicas: Ptr(int32(1)), - MaxReplicas: Ptr(int32(3)), - GroupName: "small-group", - RayStartParams: map[string]string{ - "block": "true", - }, + Replicas: Ptr(int32(1)), + MinReplicas: Ptr(int32(1)), + MaxReplicas: Ptr(int32(2)), + GroupName: "small-group", + RayStartParams: map[string]string{}, Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ InitContainers: []corev1.Container{ @@ -173,9 +153,8 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { }, Containers: []corev1.Container{ { - Name: "machine-learning", - Image: "rayproject/ray:2.0.0", - ImagePullPolicy: corev1.PullAlways, + Name: "ray-worker", + Image: "rayproject/ray:2.0.0", Lifecycle: &corev1.Lifecycle{ PreStop: &corev1.LifecycleHandler{ Exec: &corev1.ExecAction{ From 0c4e2aed7378c68583029b87b364b0fe6d3ebd33 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 22 Jun 2023 11:17:25 +0200 Subject: [PATCH 045/100] test: Add step log statements --- test/e2e/mnist_pytorch_mcad_job_test.go | 6 ++++++ test/e2e/mnist_rayjob_mcad_raycluster_test.go | 14 ++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index d305d9a6c..fcf5da437 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -58,6 +58,7 @@ func TestMNISTPyTorchMCAD(t *testing.T) { } mnistScript, err = test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), mnistScript, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created ConfigMap %s/%s successfully", mnistScript.Namespace, mnistScript.Name) // pip requirements requirements := &corev1.ConfigMap{ @@ -80,6 +81,7 @@ torchvision==0.12.0 } requirements, err = test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), requirements, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created ConfigMap %s/%s successfully", requirements.Namespace, requirements.Name) // Batch Job job := &batchv1.Job{ @@ -174,10 +176,13 @@ torchvision==0.12.0 _, err = test.Client().MCAD().ArbV1().AppWrappers(namespace.Name).Create(aw) test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created MCAD %s/%s successfully", aw.Namespace, aw.Name) + test.T().Logf("Waiting for MCAD %s/%s to be running", aw.Namespace, aw.Name) test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) + test.T().Logf("Waiting for Job %s/%s to complete successfully", job.Namespace, job.Name) test.Eventually(Job(test, namespace, job.Name), TestTimeoutLong). Should(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) @@ -191,5 +196,6 @@ torchvision==0.12.0 test.Expect(pods).To(HaveLen(1)) // Print the job logs + test.T().Logf("Printing Job %s/%s logs", job.Namespace, job.Name) test.T().Log(GetPodLogs(test, &pods[0], corev1.PodLogOptions{})) } diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index 8e8f97f8f..6bdba9984 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -42,7 +42,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { mnist, err := scripts.ReadFile("mnist.py") test.Expect(err).NotTo(HaveOccurred()) - configMap := &corev1.ConfigMap{ + mnistScript := &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1.SchemeGroupVersion.String(), Kind: "ConfigMap", @@ -56,8 +56,9 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { }, Immutable: Ptr(true), } - configMap, err = test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), configMap, metav1.CreateOptions{}) + mnistScript, err = test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), mnistScript, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created ConfigMap %s/%s successfully", mnistScript.Namespace, mnistScript.Name) // RayCluster rayCluster := &rayv1alpha1.RayCluster{ @@ -126,7 +127,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: configMap.Name, + Name: mnistScript.Name, }, }, }, @@ -212,9 +213,11 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { }, } - _, err = test.Client().MCAD().ArbV1().AppWrappers(namespace.Name).Create(aw) + aw, err = test.Client().MCAD().ArbV1().AppWrappers(namespace.Name).Create(aw) test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created MCAD %s/%s successfully", aw.Namespace, aw.Name) + test.T().Logf("Waiting for MCAD %s/%s to be running", aw.Namespace, aw.Name) test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) @@ -249,12 +252,15 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { } rayJob, err = test.Client().Ray().RayV1alpha1().RayJobs(namespace.Name).Create(test.Ctx(), rayJob, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created RayJob %s/%s successfully", rayJob.Namespace, rayJob.Name) + test.T().Logf("Waiting for RayJob %s/%s to complete successfully", rayJob.Namespace, rayJob.Name) test.Eventually(RayJob(test, namespace, rayJob.Name), TestTimeoutLong). Should(WithTransform(RayJobStatus, Equal(rayv1alpha1.JobStatusSucceeded))) rayJob, err = test.Client().Ray().RayV1alpha1().RayJobs(namespace.Name).Get(test.Ctx(), rayJob.Name, metav1.GetOptions{}) test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Printing RayJob %s/%s logs", rayJob.Namespace, rayJob.Name) test.T().Log(GetRayJobLogs(test, rayJob)) } From ae870ea3bbdadfa207cfd0c27776c5de872c235b Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 22 Jun 2023 11:57:34 +0200 Subject: [PATCH 046/100] test: Add defered troubleshooting logs --- test/e2e/mnist_pytorch_mcad_job_test.go | 30 ++++++++++++++++++++++--- test/support/batch.go | 7 +++--- test/support/core.go | 4 ++-- 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index fcf5da437..c4d6caf97 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -182,15 +182,17 @@ torchvision==0.12.0 test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) + defer troubleshooting(test, job) + test.T().Logf("Waiting for Job %s/%s to complete successfully", job.Namespace, job.Name) - test.Eventually(Job(test, namespace, job.Name), TestTimeoutLong). + test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong). Should(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) // Refresh the job to get the generated pod selector - job = GetJob(test, namespace, job.Name) + job = GetJob(test, job.Namespace, job.Name) // Get the job Pod - pods := GetPods(test, namespace, metav1.ListOptions{ + pods := GetPods(test, job.Namespace, metav1.ListOptions{ LabelSelector: labels.FormatLabels(job.Spec.Selector.MatchLabels)}, ) test.Expect(pods).To(HaveLen(1)) @@ -199,3 +201,25 @@ torchvision==0.12.0 test.T().Logf("Printing Job %s/%s logs", job.Namespace, job.Name) test.T().Log(GetPodLogs(test, &pods[0], corev1.PodLogOptions{})) } + +func troubleshooting(test Test, job *batchv1.Job) { + if !test.T().Failed() { + return + } + job = GetJob(test, job.Namespace, job.Name) + + test.T().Errorf("Job %s/%s hasn't completed in time: %s", job.Namespace, job.Name, job) + + pods := GetPods(test, job.Namespace, metav1.ListOptions{ + LabelSelector: labels.FormatLabels(job.Spec.Selector.MatchLabels)}, + ) + + if len(pods) == 0 { + test.T().Errorf("Job %s/%s has no pods scheduled", job.Namespace, job.Name) + } else { + for i, pod := range pods { + test.T().Logf("Printing Pod %s/%s logs", pod.Namespace, pod.Name) + test.T().Log(GetPodLogs(test, &pods[i], corev1.PodLogOptions{})) + } + } +} diff --git a/test/support/batch.go b/test/support/batch.go index c868d7387..2cb2d5432 100644 --- a/test/support/batch.go +++ b/test/support/batch.go @@ -20,19 +20,18 @@ import ( "github.com/onsi/gomega" batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func Job(t Test, namespace *corev1.Namespace, name string) func(g gomega.Gomega) *batchv1.Job { +func Job(t Test, namespace, name string) func(g gomega.Gomega) *batchv1.Job { return func(g gomega.Gomega) *batchv1.Job { - job, err := t.Client().Core().BatchV1().Jobs(namespace.Name).Get(t.Ctx(), name, metav1.GetOptions{}) + job, err := t.Client().Core().BatchV1().Jobs(namespace).Get(t.Ctx(), name, metav1.GetOptions{}) g.Expect(err).NotTo(gomega.HaveOccurred()) return job } } -func GetJob(t Test, namespace *corev1.Namespace, name string) *batchv1.Job { +func GetJob(t Test, namespace, name string) *batchv1.Job { t.T().Helper() return Job(t, namespace, name)(t) } diff --git a/test/support/core.go b/test/support/core.go index 9da6d0b3f..273c049a5 100644 --- a/test/support/core.go +++ b/test/support/core.go @@ -36,9 +36,9 @@ func Raw(t Test, obj runtime.Object) runtime.RawExtension { } } -func GetPods(t Test, namespace *corev1.Namespace, options metav1.ListOptions) []corev1.Pod { +func GetPods(t Test, namespace string, options metav1.ListOptions) []corev1.Pod { t.T().Helper() - pods, err := t.Client().Core().CoreV1().Pods(namespace.Name).List(t.Ctx(), options) + pods, err := t.Client().Core().CoreV1().Pods(namespace).List(t.Ctx(), options) t.Expect(err).NotTo(gomega.HaveOccurred()) return pods.Items } From 4fb8a6ffba0934ab1b54689e12ef12b7253fd59d Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 22 Jun 2023 16:07:04 +0200 Subject: [PATCH 047/100] test: Add MNIST training in RayCluster with CodeFlare SDK --- test/e2e/mnist_pytorch_mcad_job_test.go | 37 +---- test/e2e/mnist_raycluster_sdk_test.go | 145 ++++++++++++++++++ test/e2e/mnist_rayjob_mcad_raycluster_test.go | 13 +- test/e2e/requirements.txt | 1 + test/e2e/sdk.py | 39 +++++ test/e2e/support.go | 18 ++- test/support/batch.go | 24 +++ 7 files changed, 235 insertions(+), 42 deletions(-) create mode 100644 test/e2e/mnist_raycluster_sdk_test.go create mode 100644 test/e2e/requirements.txt create mode 100644 test/e2e/sdk.py diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index c4d6caf97..a704d0897 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -39,10 +39,7 @@ func TestMNISTPyTorchMCAD(t *testing.T) { namespace := test.NewTestNamespace() // MNIST training script - mnist, err := scripts.ReadFile("mnist.py") - test.Expect(err).NotTo(HaveOccurred()) - - mnistScript := &corev1.ConfigMap{ + mnist := &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1.SchemeGroupVersion.String(), Kind: "ConfigMap", @@ -52,13 +49,13 @@ func TestMNISTPyTorchMCAD(t *testing.T) { Namespace: namespace.Name, }, BinaryData: map[string][]byte{ - "mnist.py": mnist, + "mnist.py": ReadFile(test, "mnist.py"), }, Immutable: Ptr(true), } - mnistScript, err = test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), mnistScript, metav1.CreateOptions{}) + mnist, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), mnist, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created ConfigMap %s/%s successfully", mnistScript.Namespace, mnistScript.Name) + test.T().Logf("Created ConfigMap %s/%s successfully", mnist.Namespace, mnist.Name) // pip requirements requirements := &corev1.ConfigMap{ @@ -121,7 +118,7 @@ torchvision==0.12.0 VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: mnistScript.Name, + Name: mnist.Name, }, }, }, @@ -182,7 +179,7 @@ torchvision==0.12.0 test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) - defer troubleshooting(test, job) + defer JobTroubleshooting(test, job) test.T().Logf("Waiting for Job %s/%s to complete successfully", job.Namespace, job.Name) test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong). @@ -201,25 +198,3 @@ torchvision==0.12.0 test.T().Logf("Printing Job %s/%s logs", job.Namespace, job.Name) test.T().Log(GetPodLogs(test, &pods[0], corev1.PodLogOptions{})) } - -func troubleshooting(test Test, job *batchv1.Job) { - if !test.T().Failed() { - return - } - job = GetJob(test, job.Namespace, job.Name) - - test.T().Errorf("Job %s/%s hasn't completed in time: %s", job.Namespace, job.Name, job) - - pods := GetPods(test, job.Namespace, metav1.ListOptions{ - LabelSelector: labels.FormatLabels(job.Spec.Selector.MatchLabels)}, - ) - - if len(pods) == 0 { - test.T().Errorf("Job %s/%s has no pods scheduled", job.Namespace, job.Name) - } else { - for i, pod := range pods { - test.T().Logf("Printing Pod %s/%s logs", pod.Namespace, pod.Name) - test.T().Log(GetPodLogs(test, &pods[i], corev1.PodLogOptions{})) - } - } -} diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go new file mode 100644 index 000000000..7283e463b --- /dev/null +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -0,0 +1,145 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + . "github.com/onsi/gomega" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/project-codeflare/codeflare-operator/test/support" +) + +func TestMNISTRayClusterSDK(t *testing.T) { + test := With(t) + test.T().Parallel() + + test.T().Skip("Requires https://github.com/project-codeflare/codeflare-sdk/pull/146") + + // Create a namespace + namespace := test.NewTestNamespace() + + // SDK script + sdk := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "sdk", + Namespace: namespace.Name, + }, + BinaryData: map[string][]byte{ + "sdk.py": ReadFile(test, "sdk.py"), + }, + Immutable: Ptr(true), + } + sdk, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), sdk, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created ConfigMap %s/%s successfully", sdk.Namespace, sdk.Name) + + // pip requirements + requirements := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "requirements", + Namespace: namespace.Name, + }, + BinaryData: map[string][]byte{ + "requirements.txt": ReadFile(test, "requirements.txt"), + }, + Immutable: Ptr(true), + } + requirements, err = test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), requirements, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created ConfigMap %s/%s successfully", requirements.Namespace, requirements.Name) + + job := &batchv1.Job{ + TypeMeta: metav1.TypeMeta{ + APIVersion: batchv1.SchemeGroupVersion.String(), + Kind: "Job", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "sdk", + Namespace: namespace.Name, + }, + Spec: batchv1.JobSpec{ + Completions: Ptr(int32(1)), + Parallelism: Ptr(int32(1)), + BackoffLimit: Ptr(int32(0)), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "sdk", + Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", + Command: []string{"/bin/sh", "-c", "pip install -r /test/runtime/requirements.txt && python /test/job/sdk.py"}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "sdk", + MountPath: "/test/job", + }, + { + Name: "requirements", + MountPath: "/test/runtime", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "sdk", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: sdk.Name, + }, + }, + }, + }, + { + Name: "requirements", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: requirements.Name, + }, + }, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + }, + }, + }, + } + job, err = test.Client().Core().BatchV1().Jobs(namespace.Name).Create(test.Ctx(), job, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + + defer JobTroubleshooting(test, job) + + test.T().Logf("Waiting for Job %s/%s to complete successfully", job.Namespace, job.Name) + test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutMedium). + Should(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) +} diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index 6bdba9984..d7a6d18d4 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -39,10 +39,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { namespace := test.NewTestNamespace() // MNIST training script - mnist, err := scripts.ReadFile("mnist.py") - test.Expect(err).NotTo(HaveOccurred()) - - mnistScript := &corev1.ConfigMap{ + mnist := &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1.SchemeGroupVersion.String(), Kind: "ConfigMap", @@ -52,13 +49,13 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { Namespace: namespace.Name, }, BinaryData: map[string][]byte{ - "mnist.py": mnist, + "mnist.py": ReadFile(test, "mnist.py"), }, Immutable: Ptr(true), } - mnistScript, err = test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), mnistScript, metav1.CreateOptions{}) + mnist, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), mnist, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created ConfigMap %s/%s successfully", mnistScript.Namespace, mnistScript.Name) + test.T().Logf("Created ConfigMap %s/%s successfully", mnist.Namespace, mnist.Name) // RayCluster rayCluster := &rayv1alpha1.RayCluster{ @@ -127,7 +124,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: mnistScript.Name, + Name: mnist.Name, }, }, }, diff --git a/test/e2e/requirements.txt b/test/e2e/requirements.txt new file mode 100644 index 000000000..382b98555 --- /dev/null +++ b/test/e2e/requirements.txt @@ -0,0 +1 @@ +codeflare-sdk==0.4.4 diff --git a/test/e2e/sdk.py b/test/e2e/sdk.py new file mode 100644 index 000000000..578bc48c1 --- /dev/null +++ b/test/e2e/sdk.py @@ -0,0 +1,39 @@ +from codeflare_sdk.cluster.cluster import Cluster, ClusterConfiguration +# from codeflare_sdk.cluster.auth import TokenAuthentication +from codeflare_sdk.job.jobs import DDPJobDefinition + +cluster = Cluster(ClusterConfiguration( + name='mnist', + # namespace='default', + min_worker=1, + max_worker=1, + min_cpus=0.2, + max_cpus=1, + min_memory=0.5, + max_memory=1, + gpu=0, + instascale=False, +)) + +cluster.up() + +cluster.status() + +cluster.wait_ready() + +cluster.status() + +cluster.details() + +jobdef = DDPJobDefinition( + name="mnist", + script="/test/job/mnist.py", + scheduler_args={"requirements": "/test/runtime/requirements.txt"} +) +job = jobdef.submit(cluster) + +job.status() + +print(job.logs()) + +cluster.down() diff --git a/test/e2e/support.go b/test/e2e/support.go index 82f980ebd..7847c0753 100644 --- a/test/e2e/support.go +++ b/test/e2e/support.go @@ -16,7 +16,19 @@ limitations under the License. package e2e -import "embed" +import ( + "embed" -//go:embed *.py -var scripts embed.FS + "github.com/onsi/gomega" + + "github.com/project-codeflare/codeflare-operator/test/support" +) + +//go:embed *.py *.txt +var files embed.FS + +func ReadFile(t support.Test, fileName string) []byte { + file, err := files.ReadFile(fileName) + t.Expect(err).NotTo(gomega.HaveOccurred()) + return file +} diff --git a/test/support/batch.go b/test/support/batch.go index 2cb2d5432..1bf6874ea 100644 --- a/test/support/batch.go +++ b/test/support/batch.go @@ -20,7 +20,9 @@ import ( "github.com/onsi/gomega" batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" ) func Job(t Test, namespace, name string) func(g gomega.Gomega) *batchv1.Job { @@ -35,3 +37,25 @@ func GetJob(t Test, namespace, name string) *batchv1.Job { t.T().Helper() return Job(t, namespace, name)(t) } + +func JobTroubleshooting(test Test, job *batchv1.Job) { + if !test.T().Failed() { + return + } + job = GetJob(test, job.Namespace, job.Name) + + test.T().Errorf("Job %s/%s hasn't completed in time: %s", job.Namespace, job.Name, job) + + pods := GetPods(test, job.Namespace, metav1.ListOptions{ + LabelSelector: labels.FormatLabels(job.Spec.Selector.MatchLabels)}, + ) + + if len(pods) == 0 { + test.T().Errorf("Job %s/%s has no pods scheduled", job.Namespace, job.Name) + } else { + for i, pod := range pods { + test.T().Logf("Printing Pod %s/%s logs", pod.Namespace, pod.Name) + test.T().Log(GetPodLogs(test, &pods[i], corev1.PodLogOptions{})) + } + } +} From 3c211432d2fb0c4c3e96995d26c9a7e9b5670919 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 22 Jun 2023 17:25:58 +0200 Subject: [PATCH 048/100] test: Customize test timeouts --- .github/workflows/e2e_tests.yaml | 4 ++++ test/support/gomega.go | 13 ---------- test/support/support.go | 41 +++++++++++++++++++++++++++++--- 3 files changed, 42 insertions(+), 16 deletions(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 1f1a80e16..1f8263d5c 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -165,6 +165,10 @@ jobs: - name: Run e2e tests run: | + export CODEFLARE_TEST_TIMEOUT_SHORT=1m + export CODEFLARE_TEST_TIMEOUT_MEDIUM=3m + export CODEFLARE_TEST_TIMEOUT_LONG=8m + make test-e2e - name: Print CodeFlare operator logs diff --git a/test/support/gomega.go b/test/support/gomega.go index 5631044a9..74897502d 100644 --- a/test/support/gomega.go +++ b/test/support/gomega.go @@ -17,24 +17,11 @@ limitations under the License. package support import ( - "time" - "github.com/onsi/gomega" - "github.com/onsi/gomega/format" "github.com/onsi/gomega/gstruct" "github.com/onsi/gomega/types" ) -func init() { - // Gomega settings - gomega.SetDefaultEventuallyTimeout(TestTimeoutShort) - gomega.SetDefaultEventuallyPollingInterval(1 * time.Second) - gomega.SetDefaultConsistentlyDuration(30 * time.Second) - gomega.SetDefaultConsistentlyPollingInterval(1 * time.Second) - // Disable object truncation on test results - format.MaxLength = 0 -} - func EqualP(expected interface{}) types.GomegaMatcher { return gstruct.PointTo(gomega.Equal(expected)) } diff --git a/test/support/support.go b/test/support/support.go index 184b01864..782e65e7a 100644 --- a/test/support/support.go +++ b/test/support/support.go @@ -17,17 +17,52 @@ limitations under the License. package support import ( + "fmt" + "os" "time" + "github.com/onsi/gomega" + "github.com/onsi/gomega/format" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -const ( +var ( TestTimeoutShort = 1 * time.Minute TestTimeoutMedium = 2 * time.Minute TestTimeoutLong = 5 * time.Minute -) -var ( ApplyOptions = metav1.ApplyOptions{FieldManager: "codeflare-test", Force: true} ) + +func init() { + if value, ok := os.LookupEnv("CODEFLARE_TEST_TIMEOUT_SHORT"); ok { + if duration, err := time.ParseDuration(value); err == nil { + TestTimeoutShort = duration + } else { + fmt.Printf("Error parsing CODEFLARE_TEST_TIMEOUT_SHORT. Using default value: %s", TestTimeoutShort) + } + } + if value, ok := os.LookupEnv("CODEFLARE_TEST_TIMEOUT_MEDIUM"); ok { + if duration, err := time.ParseDuration(value); err == nil { + TestTimeoutMedium = duration + } else { + fmt.Printf("Error parsing CODEFLARE_TEST_TIMEOUT_MEDIUM. Using default value: %s", TestTimeoutMedium) + } + } + if value, ok := os.LookupEnv("CODEFLARE_TEST_TIMEOUT_LONG"); ok { + if duration, err := time.ParseDuration(value); err == nil { + TestTimeoutLong = duration + } else { + fmt.Printf("Error parsing CODEFLARE_TEST_TIMEOUT_LONG. Using default value: %s", TestTimeoutLong) + } + } + + // Gomega settings + gomega.SetDefaultEventuallyTimeout(TestTimeoutShort) + gomega.SetDefaultEventuallyPollingInterval(1 * time.Second) + gomega.SetDefaultConsistentlyDuration(30 * time.Second) + gomega.SetDefaultConsistentlyPollingInterval(1 * time.Second) + // Disable object truncation on test results + format.MaxLength = 0 +} From fc86a757714c6af7112abbfe88578e1cc9ca469d Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Mon, 26 Jun 2023 17:45:11 +0200 Subject: [PATCH 049/100] test: Pass MNIST training with CodeFlare SDK on OpenShift --- test/e2e/mnist.py | 16 ++- test/e2e/mnist_raycluster_sdk.py | 65 +++++++++++ test/e2e/mnist_raycluster_sdk_test.go | 150 +++++++++++++++++++------- test/e2e/requirements.txt | 4 +- test/e2e/sdk.py | 39 ------- test/support/openshift.go | 33 ++++++ 6 files changed, 225 insertions(+), 82 deletions(-) create mode 100644 test/e2e/mnist_raycluster_sdk.py delete mode 100644 test/e2e/sdk.py create mode 100644 test/support/openshift.go diff --git a/test/e2e/mnist.py b/test/e2e/mnist.py index e60ec7c6e..244c84d29 100644 --- a/test/e2e/mnist.py +++ b/test/e2e/mnist.py @@ -1,10 +1,22 @@ -# In[] +# Copyright 2022 IBM, Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import torch from pytorch_lightning import LightningModule, Trainer from pytorch_lightning.callbacks.progress import TQDMProgressBar -from pytorch_lightning.loggers import CSVLogger from torch import nn from torch.nn import functional as F from torch.utils.data import DataLoader, random_split diff --git a/test/e2e/mnist_raycluster_sdk.py b/test/e2e/mnist_raycluster_sdk.py new file mode 100644 index 000000000..444b18587 --- /dev/null +++ b/test/e2e/mnist_raycluster_sdk.py @@ -0,0 +1,65 @@ +import sys + +from time import sleep + +from torchx.specs.api import AppState, is_terminal + +from codeflare_sdk.cluster.cluster import Cluster, ClusterConfiguration +from codeflare_sdk.job.jobs import DDPJobDefinition + +namespace = sys.argv[1] + +cluster = Cluster(ClusterConfiguration( + name='mnist', + namespace=namespace, + min_worker=1, + max_worker=1, + min_cpus='500m', + max_cpus=1, + min_memory=0.5, + max_memory=1, + gpu=0, + instascale=False, +)) + +cluster.up() + +cluster.status() + +cluster.wait_ready() + +cluster.status() + +cluster.details() + +jobdef = DDPJobDefinition( + name="mnist", + script="mnist.py", + scheduler_args={"requirements": "requirements.txt"}, +) +job = jobdef.submit(cluster) + +done = False +time = 0 +timeout = 300 +while not done: + status = job.status() + if is_terminal(status.state): + break + if not done: + print(status) + if timeout and time >= timeout: + raise TimeoutError(f"job has timed out after waiting {timeout}s") + sleep(5) + time += 5 + +print(f"Job has completed: {status.state}") + +print(job.logs()) + +cluster.down() + +if not status.state == AppState.SUCCEEDED: + exit(1) +else: + exit(0) diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index 7283e463b..43854e79e 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -23,57 +23,119 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + + rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" . "github.com/project-codeflare/codeflare-operator/test/support" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" ) func TestMNISTRayClusterSDK(t *testing.T) { test := With(t) test.T().Parallel() - test.T().Skip("Requires https://github.com/project-codeflare/codeflare-sdk/pull/146") + if !IsOpenShift(test) { + test.T().Skip("Requires https://github.com/project-codeflare/codeflare-sdk/pull/146") + } // Create a namespace namespace := test.NewTestNamespace() - // SDK script - sdk := &corev1.ConfigMap{ + // Test configuration + configMap := &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1.SchemeGroupVersion.String(), Kind: "ConfigMap", }, ObjectMeta: metav1.ObjectMeta{ - Name: "sdk", + Name: "mnist-raycluster-sdk", Namespace: namespace.Name, }, BinaryData: map[string][]byte{ - "sdk.py": ReadFile(test, "sdk.py"), + // SDK script + "mnist_raycluster_sdk.py": ReadFile(test, "mnist_raycluster_sdk.py"), + // pip requirements + "requirements.txt": ReadFile(test, "requirements.txt"), + // MNIST training script + "mnist.py": ReadFile(test, "mnist.py"), }, Immutable: Ptr(true), } - sdk, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), sdk, metav1.CreateOptions{}) + configMap, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), configMap, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created ConfigMap %s/%s successfully", sdk.Namespace, sdk.Name) + test.T().Logf("Created ConfigMap %s/%s successfully", configMap.Namespace, configMap.Name) - // pip requirements - requirements := &corev1.ConfigMap{ + // SDK client RBAC + serviceAccount := &corev1.ServiceAccount{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "ConfigMap", + Kind: "ServiceAccount", }, ObjectMeta: metav1.ObjectMeta{ - Name: "requirements", + Name: "sdk-user", Namespace: namespace.Name, }, - BinaryData: map[string][]byte{ - "requirements.txt": ReadFile(test, "requirements.txt"), + } + serviceAccount, err = test.Client().Core().CoreV1().ServiceAccounts(namespace.Name).Create(test.Ctx(), serviceAccount, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + + role := &rbacv1.Role{ + TypeMeta: metav1.TypeMeta{ + APIVersion: rbacv1.SchemeGroupVersion.String(), + Kind: "Role", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "sdk", + Namespace: namespace.Name, + }, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, + APIGroups: []string{mcadv1beta1.GroupName}, + Resources: []string{"appwrappers"}, + }, + { + Verbs: []string{"get", "list"}, + APIGroups: []string{rayv1alpha1.GroupVersion.Group}, + Resources: []string{"rayclusters", "rayclusters/status"}, + }, + { + Verbs: []string{"get", "list"}, + APIGroups: []string{"route.openshift.io"}, + Resources: []string{"routes"}, + }, }, - Immutable: Ptr(true), } - requirements, err = test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), requirements, metav1.CreateOptions{}) + role, err = test.Client().Core().RbacV1().Roles(namespace.Name).Create(test.Ctx(), role, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + + roleBinding := &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{ + APIVersion: rbacv1.SchemeGroupVersion.String(), + Kind: "RoleBinding", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "sdk", + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.SchemeGroupVersion.Group, + Kind: "Role", + Name: role.Name, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + APIGroup: corev1.SchemeGroupVersion.Group, + Name: serviceAccount.Name, + Namespace: serviceAccount.Namespace, + }, + }, + } + _, err = test.Client().Core().RbacV1().RoleBindings(namespace.Name).Create(test.Ctx(), roleBinding, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created ConfigMap %s/%s successfully", requirements.Namespace, requirements.Name) job := &batchv1.Job{ TypeMeta: metav1.TypeMeta{ @@ -92,54 +154,62 @@ func TestMNISTRayClusterSDK(t *testing.T) { Spec: corev1.PodSpec{ Containers: []corev1.Container{ { - Name: "sdk", + Name: "test", Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", - Command: []string{"/bin/sh", "-c", "pip install -r /test/runtime/requirements.txt && python /test/job/sdk.py"}, + Command: []string{"/bin/sh", "-c", "pip install codeflare-sdk==0.4.4 && cp /test/* . && python mnist_raycluster_sdk.py" + " " + namespace.Name}, VolumeMounts: []corev1.VolumeMount{ { - Name: "sdk", - MountPath: "/test/job", - }, - { - Name: "requirements", - MountPath: "/test/runtime", + Name: "test", + MountPath: "/test", }, }, }, }, Volumes: []corev1.Volume{ { - Name: "sdk", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: sdk.Name, - }, - }, - }, - }, - { - Name: "requirements", + Name: "test", VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: requirements.Name, + Name: configMap.Name, }, }, }, }, }, - RestartPolicy: corev1.RestartPolicyNever, + RestartPolicy: corev1.RestartPolicyNever, + ServiceAccountName: serviceAccount.Name, }, }, }, } job, err = test.Client().Core().BatchV1().Jobs(namespace.Name).Create(test.Ctx(), job, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name) defer JobTroubleshooting(test, job) - test.T().Logf("Waiting for Job %s/%s to complete successfully", job.Namespace, job.Name) - test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutMedium). - Should(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) + test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) + test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( + Or( + WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)), + WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), + )) + + // Refresh the job to get the generated pod selector + job = GetJob(test, job.Namespace, job.Name) + + // Get the job Pod + pods := GetPods(test, job.Namespace, metav1.ListOptions{ + LabelSelector: labels.FormatLabels(job.Spec.Selector.MatchLabels)}, + ) + test.Expect(pods).To(HaveLen(1)) + + // Print the job logs + test.T().Logf("Printing Job %s/%s logs", job.Namespace, job.Name) + test.T().Log(GetPodLogs(test, &pods[0], corev1.PodLogOptions{})) + + // Assert the job has completed successfully + test.T().Logf("Checking the Job %s/%s has completed successfully", job.Namespace, job.Name) + test.Expect(job).To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) } diff --git a/test/e2e/requirements.txt b/test/e2e/requirements.txt index 382b98555..87edeef27 100644 --- a/test/e2e/requirements.txt +++ b/test/e2e/requirements.txt @@ -1 +1,3 @@ -codeflare-sdk==0.4.4 +pytorch_lightning==1.5.10 +torchmetrics==0.9.1 +torchvision==0.12.0 diff --git a/test/e2e/sdk.py b/test/e2e/sdk.py deleted file mode 100644 index 578bc48c1..000000000 --- a/test/e2e/sdk.py +++ /dev/null @@ -1,39 +0,0 @@ -from codeflare_sdk.cluster.cluster import Cluster, ClusterConfiguration -# from codeflare_sdk.cluster.auth import TokenAuthentication -from codeflare_sdk.job.jobs import DDPJobDefinition - -cluster = Cluster(ClusterConfiguration( - name='mnist', - # namespace='default', - min_worker=1, - max_worker=1, - min_cpus=0.2, - max_cpus=1, - min_memory=0.5, - max_memory=1, - gpu=0, - instascale=False, -)) - -cluster.up() - -cluster.status() - -cluster.wait_ready() - -cluster.status() - -cluster.details() - -jobdef = DDPJobDefinition( - name="mnist", - script="/test/job/mnist.py", - scheduler_args={"requirements": "/test/runtime/requirements.txt"} -) -job = jobdef.submit(cluster) - -job.status() - -print(job.logs()) - -cluster.down() diff --git a/test/support/openshift.go b/test/support/openshift.go new file mode 100644 index 000000000..cfe3b5a3f --- /dev/null +++ b/test/support/openshift.go @@ -0,0 +1,33 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package support + +import ( + "github.com/onsi/gomega" + + "k8s.io/apimachinery/pkg/api/errors" +) + +func IsOpenShift(test Test) bool { + test.T().Helper() + _, err := test.Client().Core().Discovery().ServerResourcesForGroupVersion("image.openshift.io/v1") + if err != nil && errors.IsNotFound(err) { + return false + } + test.Expect(err).NotTo(gomega.HaveOccurred()) + return true +} From f55e8c98c6250f8c1e34f89c3c8f12a93cae512e Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 27 Jun 2023 10:43:58 +0200 Subject: [PATCH 050/100] test: Print Job logs after successfull or failed completion --- test/e2e/mnist_pytorch_mcad_job_test.go | 28 ++++++++++--------------- test/e2e/mnist_raycluster_sdk_test.go | 19 +++-------------- test/support/batch.go | 17 +++++++-------- 3 files changed, 21 insertions(+), 43 deletions(-) diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index a704d0897..778fe106d 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -25,7 +25,6 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" . "github.com/project-codeflare/codeflare-operator/test/support" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" @@ -179,22 +178,17 @@ torchvision==0.12.0 test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) - defer JobTroubleshooting(test, job) - - test.T().Logf("Waiting for Job %s/%s to complete successfully", job.Namespace, job.Name) - test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong). - Should(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) - - // Refresh the job to get the generated pod selector - job = GetJob(test, job.Namespace, job.Name) - - // Get the job Pod - pods := GetPods(test, job.Namespace, metav1.ListOptions{ - LabelSelector: labels.FormatLabels(job.Spec.Selector.MatchLabels)}, - ) - test.Expect(pods).To(HaveLen(1)) + test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) + test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( + Or( + WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)), + WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), + )) // Print the job logs - test.T().Logf("Printing Job %s/%s logs", job.Namespace, job.Name) - test.T().Log(GetPodLogs(test, &pods[0], corev1.PodLogOptions{})) + PrintJobLogs(test, job.Namespace, job.Name) + + // Assert the job has completed successfully + test.Expect(GetJob(test, job.Namespace, job.Name)). + To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) } diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index 43854e79e..3761c9356 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -25,7 +25,6 @@ import ( corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" @@ -187,8 +186,6 @@ func TestMNISTRayClusterSDK(t *testing.T) { test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name) - defer JobTroubleshooting(test, job) - test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( Or( @@ -196,20 +193,10 @@ func TestMNISTRayClusterSDK(t *testing.T) { WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), )) - // Refresh the job to get the generated pod selector - job = GetJob(test, job.Namespace, job.Name) - - // Get the job Pod - pods := GetPods(test, job.Namespace, metav1.ListOptions{ - LabelSelector: labels.FormatLabels(job.Spec.Selector.MatchLabels)}, - ) - test.Expect(pods).To(HaveLen(1)) - // Print the job logs - test.T().Logf("Printing Job %s/%s logs", job.Namespace, job.Name) - test.T().Log(GetPodLogs(test, &pods[0], corev1.PodLogOptions{})) + PrintJobLogs(test, job.Namespace, job.Name) // Assert the job has completed successfully - test.T().Logf("Checking the Job %s/%s has completed successfully", job.Namespace, job.Name) - test.Expect(job).To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) + test.Expect(GetJob(test, job.Namespace, job.Name)). + To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) } diff --git a/test/support/batch.go b/test/support/batch.go index 1bf6874ea..6349efa5c 100644 --- a/test/support/batch.go +++ b/test/support/batch.go @@ -38,24 +38,21 @@ func GetJob(t Test, namespace, name string) *batchv1.Job { return Job(t, namespace, name)(t) } -func JobTroubleshooting(test Test, job *batchv1.Job) { - if !test.T().Failed() { - return - } - job = GetJob(test, job.Namespace, job.Name) +func PrintJobLogs(t Test, namespace, name string) { + t.T().Helper() - test.T().Errorf("Job %s/%s hasn't completed in time: %s", job.Namespace, job.Name, job) + job := GetJob(t, namespace, name) - pods := GetPods(test, job.Namespace, metav1.ListOptions{ + pods := GetPods(t, job.Namespace, metav1.ListOptions{ LabelSelector: labels.FormatLabels(job.Spec.Selector.MatchLabels)}, ) if len(pods) == 0 { - test.T().Errorf("Job %s/%s has no pods scheduled", job.Namespace, job.Name) + t.T().Errorf("Job %s/%s has no pods scheduled", job.Namespace, job.Name) } else { for i, pod := range pods { - test.T().Logf("Printing Pod %s/%s logs", pod.Namespace, pod.Name) - test.T().Log(GetPodLogs(test, &pods[i], corev1.PodLogOptions{})) + t.T().Logf("Printing Pod %s/%s logs", pod.Namespace, pod.Name) + t.T().Log(GetPodLogs(t, &pods[i], corev1.PodLogOptions{})) } } } From 24841ca3936c4c8c82325065af332ec76992a698 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 27 Jun 2023 10:54:09 +0200 Subject: [PATCH 051/100] test: Re-use pip requirements file --- ...rements.txt => mnist_pip_requirements.txt} | 0 test/e2e/mnist_pytorch_mcad_job_test.go | 60 ++++--------------- test/e2e/mnist_raycluster_sdk_test.go | 10 ++-- 3 files changed, 18 insertions(+), 52 deletions(-) rename test/e2e/{requirements.txt => mnist_pip_requirements.txt} (100%) diff --git a/test/e2e/requirements.txt b/test/e2e/mnist_pip_requirements.txt similarity index 100% rename from test/e2e/requirements.txt rename to test/e2e/mnist_pip_requirements.txt diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index 778fe106d..f1664a53f 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -37,47 +37,27 @@ func TestMNISTPyTorchMCAD(t *testing.T) { // Create a namespace namespace := test.NewTestNamespace() - // MNIST training script - mnist := &corev1.ConfigMap{ + // Test configuration + config := &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1.SchemeGroupVersion.String(), Kind: "ConfigMap", }, ObjectMeta: metav1.ObjectMeta{ - Name: "mnist", + Name: "mnist-mcad", Namespace: namespace.Name, }, BinaryData: map[string][]byte{ + // pip requirements + "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), + // MNIST training script "mnist.py": ReadFile(test, "mnist.py"), }, Immutable: Ptr(true), } - mnist, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), mnist, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created ConfigMap %s/%s successfully", mnist.Namespace, mnist.Name) - - // pip requirements - requirements := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "ConfigMap", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "requirements", - Namespace: namespace.Name, - }, - BinaryData: map[string][]byte{ - "requirements.txt": []byte(` -pytorch_lightning==1.5.10 -torchmetrics==0.9.1 -torchvision==0.12.0 -`), - }, - Immutable: Ptr(true), - } - requirements, err = test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), requirements, metav1.CreateOptions{}) + config, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), config, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created ConfigMap %s/%s successfully", requirements.Namespace, requirements.Name) + test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name) // Batch Job job := &batchv1.Job{ @@ -98,36 +78,22 @@ torchvision==0.12.0 { Name: "job", Image: "pytorch/pytorch:1.11.0-cuda11.3-cudnn8-runtime", - Command: []string{"/bin/sh", "-c", "pip install -r /test/runtime/requirements.txt && torchrun /test/job/mnist.py"}, + Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"}, VolumeMounts: []corev1.VolumeMount{ { - Name: "mnist", - MountPath: "/test/job", - }, - { - Name: "requirements", - MountPath: "/test/runtime", + Name: "test", + MountPath: "/test", }, }, }, }, Volumes: []corev1.Volume{ { - Name: "mnist", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: mnist.Name, - }, - }, - }, - }, - { - Name: "requirements", + Name: "test", VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: requirements.Name, + Name: config.Name, }, }, }, diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index 3761c9356..820871550 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -44,7 +44,7 @@ func TestMNISTRayClusterSDK(t *testing.T) { namespace := test.NewTestNamespace() // Test configuration - configMap := &corev1.ConfigMap{ + config := &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1.SchemeGroupVersion.String(), Kind: "ConfigMap", @@ -57,15 +57,15 @@ func TestMNISTRayClusterSDK(t *testing.T) { // SDK script "mnist_raycluster_sdk.py": ReadFile(test, "mnist_raycluster_sdk.py"), // pip requirements - "requirements.txt": ReadFile(test, "requirements.txt"), + "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), // MNIST training script "mnist.py": ReadFile(test, "mnist.py"), }, Immutable: Ptr(true), } - configMap, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), configMap, metav1.CreateOptions{}) + config, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), config, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created ConfigMap %s/%s successfully", configMap.Namespace, configMap.Name) + test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name) // SDK client RBAC serviceAccount := &corev1.ServiceAccount{ @@ -170,7 +170,7 @@ func TestMNISTRayClusterSDK(t *testing.T) { VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: configMap.Name, + Name: config.Name, }, }, }, From 475137e4e2b37eb8b786395aa81a09e15d2343f8 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 27 Jun 2023 15:20:26 +0200 Subject: [PATCH 052/100] test: Parameterize CodeFlare SDK version --- Makefile | 4 ++++ controllers/defaults.go | 5 +++-- test/e2e/mnist_raycluster_sdk_test.go | 2 +- test/support/codeflare.go | 30 +++++++++++++++++++++++++++ 4 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 test/support/codeflare.go diff --git a/Makefile b/Makefile index c9295f57e..2185264ee 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,9 @@ MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_REF} # KUBERAY_VERSION defines the default version of the KubeRay operator KUBERAY_VERSION ?= v0.5.0 +# CODEFLARE_SDK_VERSION defines the default version of the CodeFlare SDK +CODEFLARE_SDK_VERSION ?= 0.4.4 + # OPERATORS_REPO_ORG points to GitHub repository organization where bundle PR is opened against # OPERATORS_REPO_FORK_ORG points to GitHub repository fork organization where bundle build is pushed to OPERATORS_REPO_ORG ?= redhat-openshift-ecosystem @@ -132,6 +135,7 @@ defaults: @echo "const (" >> $(DEFAULTS_FILE) @echo " MCADImage = \"$(MCAD_IMAGE)\"" >> $(DEFAULTS_FILE) @echo " InstaScaleImage = \"$(INSTASCALE_IMAGE)\"" >> $(DEFAULTS_FILE) + @echo " CodeFlareSDKVersion = \"$(CODEFLARE_SDK_VERSION)\"" >> $(DEFAULTS_FILE) @echo "" >> $(DEFAULTS_FILE) @echo ")" >> $(DEFAULTS_FILE) @echo "" >> $(DEFAULTS_FILE) diff --git a/controllers/defaults.go b/controllers/defaults.go index d3cac3d07..497940a94 100644 --- a/controllers/defaults.go +++ b/controllers/defaults.go @@ -5,6 +5,7 @@ package controllers // *********************** const ( - MCADImage = "quay.io/project-codeflare/mcad-controller:release-v1.31.0" - InstaScaleImage = "quay.io/project-codeflare/instascale-controller:v0.0.4" + MCADImage = "quay.io/project-codeflare/mcad-controller:release-v1.31.0" + InstaScaleImage = "quay.io/project-codeflare/instascale-controller:v0.0.4" + CodeFlareSDKVersion = "0.4.4" ) diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index 820871550..3ba168a0f 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -155,7 +155,7 @@ func TestMNISTRayClusterSDK(t *testing.T) { { Name: "test", Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", - Command: []string{"/bin/sh", "-c", "pip install codeflare-sdk==0.4.4 && cp /test/* . && python mnist_raycluster_sdk.py" + " " + namespace.Name}, + Command: []string{"/bin/sh", "-c", "pip install codeflare-sdk==" + GetCodeFlareSDKVersion() + " && cp /test/* . && python mnist_raycluster_sdk.py" + " " + namespace.Name}, VolumeMounts: []corev1.VolumeMount{ { Name: "test", diff --git a/test/support/codeflare.go b/test/support/codeflare.go new file mode 100644 index 000000000..69066c2a5 --- /dev/null +++ b/test/support/codeflare.go @@ -0,0 +1,30 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package support + +import ( + "os" + + "github.com/project-codeflare/codeflare-operator/controllers" +) + +func GetCodeFlareSDKVersion() string { + if value, ok := os.LookupEnv("CODEFLARE_SDK_VERSION"); ok { + return value + } + return controllers.CodeFlareSDKVersion +} From f0960c1530d4d3782c8b4238c9a0915019960e23 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 27 Jun 2023 15:58:06 +0200 Subject: [PATCH 053/100] test: Remove ray_lightning from requirements --- test/e2e/mnist_rayjob_mcad_raycluster_test.go | 1 - test/support/ray.go | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index d7a6d18d4..38a3b97c9 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -233,7 +233,6 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { { "pip": [ "pytorch_lightning==1.5.10", - "ray_lightning", "torchmetrics==0.9.1", "torchvision==0.12.0" ], diff --git a/test/support/ray.go b/test/support/ray.go index 369db80bf..9e0d22d94 100644 --- a/test/support/ray.go +++ b/test/support/ray.go @@ -45,7 +45,8 @@ func GetRayJobLogs(t Test, job *rayv1alpha1.RayJob) string { t.T().Helper() response := t.Client().Core().CoreV1().RESTClient(). Get(). - AbsPath("/api/v1/namespaces", job.Namespace, "services", "http:"+job.Status.RayClusterName+"-head-svc:dashboard", "proxy", "api", "jobs", job.Status.JobId, "logs").Do(t.Ctx()) + AbsPath("/api/v1/namespaces", job.Namespace, "services", "http:"+job.Status.RayClusterName+"-head-svc:dashboard", "proxy", "api", "jobs", job.Status.JobId, "logs"). + Do(t.Ctx()) t.Expect(response.Error()).NotTo(gomega.HaveOccurred()) body := map[string]string{} From 3fd6af66dc84ad5c2033bf34579b5693897032f5 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 27 Jun 2023 16:35:59 +0200 Subject: [PATCH 054/100] test: Parameterize Ray image and version --- Makefile | 27 ++++++++++++++++--- controllers/defaults.go | 5 ++-- test/e2e/mnist_rayjob_mcad_raycluster_test.go | 6 ++--- test/support/codeflare.go | 26 +++++++++++++++--- test/support/defaults.go | 11 ++++++++ 5 files changed, 62 insertions(+), 13 deletions(-) create mode 100644 test/support/defaults.go diff --git a/Makefile b/Makefile index 2185264ee..c3ce86656 100644 --- a/Makefile +++ b/Makefile @@ -21,9 +21,12 @@ MCAD_REF ?= release-${MCAD_VERSION} MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_REF} -# KUBERAY_VERSION defines the default version of the KubeRay operator +# KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) KUBERAY_VERSION ?= v0.5.0 +# RAY_VERSION defines the default version of Ray (used for testing) +RAY_VERSION ?= 2.5.0 + # CODEFLARE_SDK_VERSION defines the default version of the CodeFlare SDK CODEFLARE_SDK_VERSION ?= 0.4.4 @@ -67,6 +70,9 @@ MCAD_IMAGE ?= $(IMAGE_ORG_BASE)/mcad-controller:$(MCAD_REF) # INSTASCALE_IMAGE defines the default container image for the InstaScale controller INSTASCALE_IMAGE ?= $(IMAGE_ORG_BASE)/instascale-controller:$(INSTASCALE_VERSION) +# RAY_IMAGE defines the default container image for Ray (used for testing) +RAY_IMAGE ?= rayproject/ray:$(RAY_VERSION) + # BUNDLE_IMG defines the image:tag used for the bundle. # You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:$(VERSION) @@ -122,6 +128,7 @@ help: ## Display this help. ##@ Development DEFAULTS_FILE := controllers/defaults.go +DEFAULTS_TEST_FILE := test/support/defaults.go .PHONY: defaults defaults: @@ -135,12 +142,26 @@ defaults: @echo "const (" >> $(DEFAULTS_FILE) @echo " MCADImage = \"$(MCAD_IMAGE)\"" >> $(DEFAULTS_FILE) @echo " InstaScaleImage = \"$(INSTASCALE_IMAGE)\"" >> $(DEFAULTS_FILE) - @echo " CodeFlareSDKVersion = \"$(CODEFLARE_SDK_VERSION)\"" >> $(DEFAULTS_FILE) @echo "" >> $(DEFAULTS_FILE) @echo ")" >> $(DEFAULTS_FILE) @echo "" >> $(DEFAULTS_FILE) - gofmt -w $(DEFAULTS_FILE) + $(info Regenerating $(DEFAULTS_TEST_FILE)) + @echo "package support" > $(DEFAULTS_TEST_FILE) + @echo "" >> $(DEFAULTS_TEST_FILE) + @echo "// ***********************" >> $(DEFAULTS_TEST_FILE) + @echo "// DO NOT EDIT THIS FILE" >> $(DEFAULTS_TEST_FILE) + @echo "// ***********************" >> $(DEFAULTS_TEST_FILE) + @echo "" >> $(DEFAULTS_TEST_FILE) + @echo "const (" >> $(DEFAULTS_TEST_FILE) + @echo " CodeFlareSDKVersion = \"$(CODEFLARE_SDK_VERSION)\"" >> $(DEFAULTS_TEST_FILE) + @echo " RayVersion = \"$(RAY_VERSION)\"" >> $(DEFAULTS_TEST_FILE) + @echo " RayImage = \"$(RAY_IMAGE)\"" >> $(DEFAULTS_TEST_FILE) + @echo "" >> $(DEFAULTS_TEST_FILE) + @echo ")" >> $(DEFAULTS_TEST_FILE) + @echo "" >> $(DEFAULTS_TEST_FILE) + + gofmt -w $(DEFAULTS_FILE) $(DEFAULTS_TEST_FILE) .PHONY: manifests manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. diff --git a/controllers/defaults.go b/controllers/defaults.go index 497940a94..d3cac3d07 100644 --- a/controllers/defaults.go +++ b/controllers/defaults.go @@ -5,7 +5,6 @@ package controllers // *********************** const ( - MCADImage = "quay.io/project-codeflare/mcad-controller:release-v1.31.0" - InstaScaleImage = "quay.io/project-codeflare/instascale-controller:v0.0.4" - CodeFlareSDKVersion = "0.4.4" + MCADImage = "quay.io/project-codeflare/mcad-controller:release-v1.31.0" + InstaScaleImage = "quay.io/project-codeflare/instascale-controller:v0.0.4" ) diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index 38a3b97c9..33a9e6a51 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -68,7 +68,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { Namespace: namespace.Name, }, Spec: rayv1alpha1.RayClusterSpec{ - RayVersion: "2.0.0", + RayVersion: GetRayVersion(), HeadGroupSpec: rayv1alpha1.HeadGroupSpec{ RayStartParams: map[string]string{ "dashboard-host": "0.0.0.0", @@ -78,7 +78,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { Containers: []corev1.Container{ { Name: "ray-head", - Image: "rayproject/ray:2.0.0", + Image: GetRayImage(), Ports: []corev1.ContainerPort{ { ContainerPort: 6379, @@ -152,7 +152,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { Containers: []corev1.Container{ { Name: "ray-worker", - Image: "rayproject/ray:2.0.0", + Image: GetRayImage(), Lifecycle: &corev1.Lifecycle{ PreStop: &corev1.LifecycleHandler{ Exec: &corev1.ExecAction{ diff --git a/test/support/codeflare.go b/test/support/codeflare.go index 69066c2a5..0763bbd34 100644 --- a/test/support/codeflare.go +++ b/test/support/codeflare.go @@ -18,13 +18,31 @@ package support import ( "os" +) - "github.com/project-codeflare/codeflare-operator/controllers" +// The environment variables hereafter can be used to change the components +// used for testing. +const ( + CodeFlareTestSdkVersion = "CODEFLARE_TEST_SDK_VERSION" + CodeFlareTestRayVersion = "CODEFLARE_TEST_RAY_VERSION" + CodeFlareTestRayImage = "CODEFLARE_TEST_RAY_IMAGE" ) func GetCodeFlareSDKVersion() string { - if value, ok := os.LookupEnv("CODEFLARE_SDK_VERSION"); ok { - return value + return lookupEnvOrDefault(CodeFlareTestSdkVersion, CodeFlareSDKVersion) +} + +func GetRayVersion() string { + return lookupEnvOrDefault(CodeFlareTestRayVersion, RayVersion) +} + +func GetRayImage() string { + return lookupEnvOrDefault(CodeFlareTestRayImage, RayImage) +} + +func lookupEnvOrDefault(key, value string) string { + if v, ok := os.LookupEnv(key); ok { + return v } - return controllers.CodeFlareSDKVersion + return value } diff --git a/test/support/defaults.go b/test/support/defaults.go new file mode 100644 index 000000000..a0aaad421 --- /dev/null +++ b/test/support/defaults.go @@ -0,0 +1,11 @@ +package support + +// *********************** +// DO NOT EDIT THIS FILE +// *********************** + +const ( + CodeFlareSDKVersion = "0.4.4" + RayVersion = "2.5.0" + RayImage = "rayproject/ray:2.5.0" +) From 2d477f6fc8d5a287cc3efe6f5133941aad1525af Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 27 Jun 2023 16:40:36 +0200 Subject: [PATCH 055/100] test: Parameterize PyTorch image --- test/e2e/mnist_pytorch_mcad_job_test.go | 2 +- test/support/codeflare.go | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index f1664a53f..1b1125af4 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -77,7 +77,7 @@ func TestMNISTPyTorchMCAD(t *testing.T) { Containers: []corev1.Container{ { Name: "job", - Image: "pytorch/pytorch:1.11.0-cuda11.3-cudnn8-runtime", + Image: GetPyTorchImage(), Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"}, VolumeMounts: []corev1.VolumeMount{ { diff --git a/test/support/codeflare.go b/test/support/codeflare.go index 0763bbd34..488a8fffd 100644 --- a/test/support/codeflare.go +++ b/test/support/codeflare.go @@ -23,9 +23,10 @@ import ( // The environment variables hereafter can be used to change the components // used for testing. const ( - CodeFlareTestSdkVersion = "CODEFLARE_TEST_SDK_VERSION" - CodeFlareTestRayVersion = "CODEFLARE_TEST_RAY_VERSION" - CodeFlareTestRayImage = "CODEFLARE_TEST_RAY_IMAGE" + CodeFlareTestSdkVersion = "CODEFLARE_TEST_SDK_VERSION" + CodeFlareTestRayVersion = "CODEFLARE_TEST_RAY_VERSION" + CodeFlareTestRayImage = "CODEFLARE_TEST_RAY_IMAGE" + CodeFlareTestPyTorchImage = "CODEFLARE_TEST_PYTORCH_IMAGE" ) func GetCodeFlareSDKVersion() string { @@ -40,6 +41,10 @@ func GetRayImage() string { return lookupEnvOrDefault(CodeFlareTestRayImage, RayImage) } +func GetPyTorchImage() string { + return lookupEnvOrDefault(CodeFlareTestPyTorchImage, "pytorch/pytorch:1.11.0-cuda11.3-cudnn8-runtime") +} + func lookupEnvOrDefault(key, value string) string { if v, ok := os.LookupEnv(key); ok { return v From 57b427d7c88bb4cc45a7ee52933339f26b64472c Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 27 Jun 2023 16:42:48 +0200 Subject: [PATCH 056/100] test: Add FIXME for SDK user base image --- test/e2e/mnist_raycluster_sdk_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index 3ba168a0f..d919fdcf9 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -153,7 +153,9 @@ func TestMNISTRayClusterSDK(t *testing.T) { Spec: corev1.PodSpec{ Containers: []corev1.Container{ { - Name: "test", + Name: "test", + // FIXME: switch to base Python image once the dependency on OpenShift CLI is removed + // See https://github.com/project-codeflare/codeflare-sdk/pull/146 Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", Command: []string{"/bin/sh", "-c", "pip install codeflare-sdk==" + GetCodeFlareSDKVersion() + " && cp /test/* . && python mnist_raycluster_sdk.py" + " " + namespace.Name}, VolumeMounts: []corev1.VolumeMount{ From 9c79ab0a9d38af1fcadf1a9ed2c05b7a212279cc Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 27 Jun 2023 17:35:38 +0200 Subject: [PATCH 057/100] Align go.mod with MCAD version --- Makefile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c3ce86656..f70edf1e3 100644 --- a/Makefile +++ b/Makefile @@ -214,12 +214,16 @@ vet: ## Run go vet against code. ##@ Build +.PHONY: modules +modules: ## Update Go dependencies. + go get $(MCAD_REPO)@$(MCAD_VERSION) + .PHONY: build -build: defaults generate fmt vet ## Build manager binary. +build: modules defaults generate fmt vet ## Build manager binary. go build -o bin/manager main.go .PHONY: run -run: defaults manifests generate fmt vet ## Run a controller from your host. +run: modules defaults manifests generate fmt vet ## Run a controller from your host. go run ./main.go .PHONY: image-build From f7b7a3a0dcbf804e699223b4889fcddc3846e425 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 28 Jun 2023 15:49:46 +0200 Subject: [PATCH 058/100] test: Print Ray job logs after successful or failed completion --- test/e2e/mnist_rayjob_mcad_raycluster_test.go | 15 ++++++++------- test/support/ray.go | 15 +++++++++++---- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index 33a9e6a51..024dee924 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -250,13 +250,14 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created RayJob %s/%s successfully", rayJob.Namespace, rayJob.Name) - test.T().Logf("Waiting for RayJob %s/%s to complete successfully", rayJob.Namespace, rayJob.Name) - test.Eventually(RayJob(test, namespace, rayJob.Name), TestTimeoutLong). - Should(WithTransform(RayJobStatus, Equal(rayv1alpha1.JobStatusSucceeded))) - - rayJob, err = test.Client().Ray().RayV1alpha1().RayJobs(namespace.Name).Get(test.Ctx(), rayJob.Name, metav1.GetOptions{}) - test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Waiting for RayJob %s/%s to complete", rayJob.Namespace, rayJob.Name) + test.Eventually(RayJob(test, rayJob.Namespace, rayJob.Name), TestTimeoutLong). + Should(WithTransform(RayJobStatus, Satisfy(rayv1alpha1.IsJobTerminal))) test.T().Logf("Printing RayJob %s/%s logs", rayJob.Namespace, rayJob.Name) - test.T().Log(GetRayJobLogs(test, rayJob)) + test.T().Log(GetRayJobLogs(test, rayJob.Namespace, rayJob.Name)) + + // Assert the Ray job has completed successfully + test.Expect(GetRayJob(test, rayJob.Namespace, rayJob.Name)). + To(WithTransform(RayJobStatus, Equal(rayv1alpha1.JobStatusSucceeded))) } diff --git a/test/support/ray.go b/test/support/ray.go index 9e0d22d94..75d775701 100644 --- a/test/support/ray.go +++ b/test/support/ray.go @@ -21,7 +21,6 @@ import ( "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" @@ -29,20 +28,28 @@ import ( const RayJobDefaultClusterSelectorKey = "ray.io/cluster" -func RayJob(t Test, namespace *corev1.Namespace, name string) func(g gomega.Gomega) *rayv1alpha1.RayJob { +func RayJob(t Test, namespace, name string) func(g gomega.Gomega) *rayv1alpha1.RayJob { return func(g gomega.Gomega) *rayv1alpha1.RayJob { - job, err := t.Client().Ray().RayV1alpha1().RayJobs(namespace.Name).Get(t.Ctx(), name, metav1.GetOptions{}) + job, err := t.Client().Ray().RayV1alpha1().RayJobs(namespace).Get(t.Ctx(), name, metav1.GetOptions{}) g.Expect(err).NotTo(gomega.HaveOccurred()) return job } } +func GetRayJob(t Test, namespace, name string) *rayv1alpha1.RayJob { + t.T().Helper() + return RayJob(t, namespace, name)(t) +} + func RayJobStatus(job *rayv1alpha1.RayJob) rayv1alpha1.JobStatus { return job.Status.JobStatus } -func GetRayJobLogs(t Test, job *rayv1alpha1.RayJob) string { +func GetRayJobLogs(t Test, namespace, name string) string { t.T().Helper() + + job := GetRayJob(t, namespace, name) + response := t.Client().Core().CoreV1().RESTClient(). Get(). AbsPath("/api/v1/namespaces", job.Namespace, "services", "http:"+job.Status.RayClusterName+"-head-svc:dashboard", "proxy", "api", "jobs", job.Status.JobId, "logs"). From 321a8aaec4c5239da47b875539caad6484e7acc3 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 29 Jun 2023 11:18:42 +0200 Subject: [PATCH 059/100] test: Upload job logs --- .github/workflows/e2e_tests.yaml | 12 ++++++ test/e2e/mnist_pytorch_mcad_job_test.go | 4 +- test/e2e/mnist_raycluster_sdk_test.go | 4 +- test/e2e/mnist_rayjob_mcad_raycluster_test.go | 4 +- test/e2e/support.go | 1 + test/support/batch.go | 6 +-- test/support/codeflare.go | 9 +++- test/support/core.go | 4 +- test/support/ray.go | 4 +- test/support/test.go | 41 +++++++++++++++++-- test/support/utils.go | 20 +++++++++ 11 files changed, 90 insertions(+), 19 deletions(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 1f8263d5c..8d9318c75 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -169,6 +169,9 @@ jobs: export CODEFLARE_TEST_TIMEOUT_MEDIUM=3m export CODEFLARE_TEST_TIMEOUT_LONG=8m + export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} + echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV + make test-e2e - name: Print CodeFlare operator logs @@ -188,3 +191,12 @@ jobs: run: | echo "Printing KubeRay operator logs" kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay + + - name: Upload logs + uses: actions/upload-artifact@v3 + if: always() && steps.deploy.outcome == 'success' + with: + name: logs + retention-days: 10 + path: | + ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/**/*.log diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index 1b1125af4..07bcf95a6 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -151,8 +151,8 @@ func TestMNISTPyTorchMCAD(t *testing.T) { WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), )) - // Print the job logs - PrintJobLogs(test, job.Namespace, job.Name) + // Retrieve the job logs + WriteJobLogs(test, job.Namespace, job.Name) // Assert the job has completed successfully test.Expect(GetJob(test, job.Namespace, job.Name)). diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index d919fdcf9..4e804fd5b 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -195,8 +195,8 @@ func TestMNISTRayClusterSDK(t *testing.T) { WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), )) - // Print the job logs - PrintJobLogs(test, job.Namespace, job.Name) + // Retrieve the job logs + WriteJobLogs(test, job.Namespace, job.Name) // Assert the job has completed successfully test.Expect(GetJob(test, job.Namespace, job.Name)). diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index 024dee924..f20ba0a7a 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -254,8 +254,8 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { test.Eventually(RayJob(test, rayJob.Namespace, rayJob.Name), TestTimeoutLong). Should(WithTransform(RayJobStatus, Satisfy(rayv1alpha1.IsJobTerminal))) - test.T().Logf("Printing RayJob %s/%s logs", rayJob.Namespace, rayJob.Name) - test.T().Log(GetRayJobLogs(test, rayJob.Namespace, rayJob.Name)) + test.T().Logf("Retrieving RayJob %s/%s logs", rayJob.Namespace, rayJob.Name) + WriteToOutputDir(test, rayJob.Name, Log, GetRayJobLogs(test, rayJob.Namespace, rayJob.Name)) // Assert the Ray job has completed successfully test.Expect(GetRayJob(test, rayJob.Namespace, rayJob.Name)). diff --git a/test/e2e/support.go b/test/e2e/support.go index 7847c0753..14bf19ce4 100644 --- a/test/e2e/support.go +++ b/test/e2e/support.go @@ -28,6 +28,7 @@ import ( var files embed.FS func ReadFile(t support.Test, fileName string) []byte { + t.T().Helper() file, err := files.ReadFile(fileName) t.Expect(err).NotTo(gomega.HaveOccurred()) return file diff --git a/test/support/batch.go b/test/support/batch.go index 6349efa5c..db8feeb53 100644 --- a/test/support/batch.go +++ b/test/support/batch.go @@ -38,7 +38,7 @@ func GetJob(t Test, namespace, name string) *batchv1.Job { return Job(t, namespace, name)(t) } -func PrintJobLogs(t Test, namespace, name string) { +func WriteJobLogs(t Test, namespace, name string) { t.T().Helper() job := GetJob(t, namespace, name) @@ -51,8 +51,8 @@ func PrintJobLogs(t Test, namespace, name string) { t.T().Errorf("Job %s/%s has no pods scheduled", job.Namespace, job.Name) } else { for i, pod := range pods { - t.T().Logf("Printing Pod %s/%s logs", pod.Namespace, pod.Name) - t.T().Log(GetPodLogs(t, &pods[i], corev1.PodLogOptions{})) + t.T().Logf("Retrieving Pod %s/%s logs", pod.Namespace, pod.Name) + WriteToOutputDir(t, pod.Name, Log, GetPodLogs(t, &pods[i], corev1.PodLogOptions{})) } } } diff --git a/test/support/codeflare.go b/test/support/codeflare.go index 488a8fffd..04b1f3e96 100644 --- a/test/support/codeflare.go +++ b/test/support/codeflare.go @@ -20,13 +20,18 @@ import ( "os" ) -// The environment variables hereafter can be used to change the components -// used for testing. const ( + // The environment variables hereafter can be used to change the components + // used for testing. + CodeFlareTestSdkVersion = "CODEFLARE_TEST_SDK_VERSION" CodeFlareTestRayVersion = "CODEFLARE_TEST_RAY_VERSION" CodeFlareTestRayImage = "CODEFLARE_TEST_RAY_IMAGE" CodeFlareTestPyTorchImage = "CODEFLARE_TEST_PYTORCH_IMAGE" + + // The testing output directory, to write output files into. + + CodeFlareTestOutputDir = "CODEFLARE_TEST_OUTPUT_DIR" ) func GetCodeFlareSDKVersion() string { diff --git a/test/support/core.go b/test/support/core.go index 273c049a5..a9be18bfa 100644 --- a/test/support/core.go +++ b/test/support/core.go @@ -43,7 +43,7 @@ func GetPods(t Test, namespace string, options metav1.ListOptions) []corev1.Pod return pods.Items } -func GetPodLogs(t Test, pod *corev1.Pod, options corev1.PodLogOptions) string { +func GetPodLogs(t Test, pod *corev1.Pod, options corev1.PodLogOptions) []byte { t.T().Helper() stream, err := t.Client().Core().CoreV1().Pods(pod.GetNamespace()).GetLogs(pod.GetName(), &options).Stream(t.Ctx()) t.Expect(err).NotTo(gomega.HaveOccurred()) @@ -55,5 +55,5 @@ func GetPodLogs(t Test, pod *corev1.Pod, options corev1.PodLogOptions) string { bytes, err := io.ReadAll(stream) t.Expect(err).NotTo(gomega.HaveOccurred()) - return string(bytes) + return bytes } diff --git a/test/support/ray.go b/test/support/ray.go index 75d775701..3833d43b7 100644 --- a/test/support/ray.go +++ b/test/support/ray.go @@ -45,7 +45,7 @@ func RayJobStatus(job *rayv1alpha1.RayJob) rayv1alpha1.JobStatus { return job.Status.JobStatus } -func GetRayJobLogs(t Test, namespace, name string) string { +func GetRayJobLogs(t Test, namespace, name string) []byte { t.T().Helper() job := GetRayJob(t, namespace, name) @@ -61,5 +61,5 @@ func GetRayJobLogs(t Test, namespace, name string) string { t.Expect(json.Unmarshal(bytes, &body)).To(gomega.Succeed()) t.Expect(body).To(gomega.HaveKey("logs")) - return body["logs"] + return []byte(body["logs"]) } diff --git a/test/support/test.go b/test/support/test.go index 5b3b271b7..75556ca40 100644 --- a/test/support/test.go +++ b/test/support/test.go @@ -18,6 +18,8 @@ package support import ( "context" + "os" + "path" "sync" "testing" @@ -30,6 +32,7 @@ type Test interface { T() *testing.T Ctx() context.Context Client() Client + OutputDir() string gomega.Gomega @@ -70,9 +73,13 @@ type T struct { *gomega.WithT t *testing.T // nolint: containedctx - ctx context.Context - client Client - once sync.Once + ctx context.Context + client Client + outputDir string + once struct { + client sync.Once + outputDir sync.Once + } } func (t *T) T() *testing.T { @@ -84,7 +91,8 @@ func (t *T) Ctx() context.Context { } func (t *T) Client() Client { - t.once.Do(func() { + t.T().Helper() + t.once.client.Do(func() { c, err := newTestClient() if err != nil { t.T().Fatalf("Error creating client: %v", err) @@ -94,6 +102,31 @@ func (t *T) Client() Client { return t.client } +func (t *T) OutputDir() string { + t.T().Helper() + t.once.outputDir.Do(func() { + if parent, ok := os.LookupEnv(CodeFlareTestOutputDir); ok { + if !path.IsAbs(parent) { + if cwd, err := os.Getwd(); err == nil { + // best effort to output the parent absolute path + parent = path.Join(cwd, parent) + } + } + t.T().Logf("Creating output directory in parent directory: %s", parent) + dir, err := os.MkdirTemp(parent, t.T().Name()) + if err != nil { + t.T().Fatalf("Error creating output directory: %v", err) + } + t.outputDir = dir + } else { + t.T().Logf("Creating ephemeral output directory as %s env variable is unset", CodeFlareTestOutputDir) + t.outputDir = t.T().TempDir() + } + t.T().Logf("Output directory has been created at: %s", t.outputDir) + }) + return t.outputDir +} + func (t *T) NewTestNamespace(options ...Option[*corev1.Namespace]) *corev1.Namespace { t.T().Helper() namespace := createTestNamespace(t, options...) diff --git a/test/support/utils.go b/test/support/utils.go index ed40309f2..595ac4103 100644 --- a/test/support/utils.go +++ b/test/support/utils.go @@ -16,6 +16,26 @@ limitations under the License. package support +import ( + "io/fs" + "os" + "path" + + "github.com/onsi/gomega" +) + func Ptr[T any](v T) *T { return &v } + +type OutputType string + +const ( + Log OutputType = "log" +) + +func WriteToOutputDir(t Test, fileName string, fileType OutputType, data []byte) { + t.T().Helper() + t.Expect(os.WriteFile(path.Join(t.OutputDir(), fileName+"."+string(fileType)), data, fs.ModePerm)). + To(gomega.Succeed()) +} From 630199b0fd11685c6ea23fe6ea705e25dd913229 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 29 Jun 2023 13:35:05 +0200 Subject: [PATCH 060/100] test Remove unused functions --- test/support/gomega.go | 31 ------------------------------- 1 file changed, 31 deletions(-) delete mode 100644 test/support/gomega.go diff --git a/test/support/gomega.go b/test/support/gomega.go deleted file mode 100644 index 74897502d..000000000 --- a/test/support/gomega.go +++ /dev/null @@ -1,31 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "github.com/onsi/gomega" - "github.com/onsi/gomega/gstruct" - "github.com/onsi/gomega/types" -) - -func EqualP(expected interface{}) types.GomegaMatcher { - return gstruct.PointTo(gomega.Equal(expected)) -} - -func MatchFieldsP(options gstruct.Options, fields gstruct.Fields) types.GomegaMatcher { - return gstruct.PointTo(gstruct.MatchFields(options, fields)) -} From 9771055b8d28c1c3e51e0039a68abe6791665e19 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 30 Jun 2023 14:56:33 +0200 Subject: [PATCH 061/100] test: Fix Unexpected kind-action input --- .github/workflows/e2e_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 8d9318c75..2783c52b7 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -90,7 +90,7 @@ jobs: - name: Setup KinD cluster uses: helm/kind-action@v1.5.0 with: - name: cluster + cluster_name: cluster version: v0.17.0 config: ${{ env.KIND_CONFIG_FILE }} From 06d47ed34464e6ac963ef9ee33f078015b5caf86 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 30 Jun 2023 15:14:18 +0200 Subject: [PATCH 062/100] test: Format test output using gotestfmt --- .github/workflows/e2e_tests.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 2783c52b7..1ec1cd0ac 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -68,6 +68,11 @@ jobs: with: go-version: v1.18 + - name: Set up gotestfmt + uses: gotesttools/gotestfmt-action@v2 + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: Container image registry run: | podman run -d -p 5000:5000 --name registry registry:2.8.1 @@ -172,7 +177,8 @@ jobs: export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV - make test-e2e + set -euo pipefail + go test -timeout 30m -v ./test/e2e -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt - name: Print CodeFlare operator logs if: always() && steps.deploy.outcome == 'success' From d665dcc325db0e4a0848e8576cd9876222ed8ae4 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 30 Jun 2023 15:52:00 +0200 Subject: [PATCH 063/100] test: Add codeflare stack logs to uploaded artifacts --- .github/workflows/e2e_tests.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 1ec1cd0ac..0c6bb2b91 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -184,19 +184,19 @@ jobs: if: always() && steps.deploy.outcome == 'success' run: | echo "Printing CodeFlare operator logs" - kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator + kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator | tee ${CODEFLARE_TEST_OUTPUT_DIR}/codeflare-operator.log - name: Print MCAD controller logs if: always() && steps.deploy.outcome == 'success' run: | echo "Printing MCAD controller logs" - kubectl logs -n codeflare-system --tail -1 -l component=multi-cluster-application-dispatcher + kubectl logs -n codeflare-system --tail -1 -l component=multi-cluster-application-dispatcher | tee ${CODEFLARE_TEST_OUTPUT_DIR}/mcad.log - name: Print KubeRay operator logs if: always() && steps.deploy.outcome == 'success' run: | echo "Printing KubeRay operator logs" - kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay + kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kuberay.log - name: Upload logs uses: actions/upload-artifact@v3 From 9d536567b126006f5c7496f970cb780241a6eae4 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 30 Jun 2023 16:44:06 +0200 Subject: [PATCH 064/100] test: Add description to e2e tests --- test/e2e/mnist_pytorch_mcad_job_test.go | 1 + test/e2e/mnist_raycluster_sdk_test.go | 6 ++++++ test/e2e/mnist_rayjob_mcad_raycluster_test.go | 2 ++ 3 files changed, 9 insertions(+) diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index 07bcf95a6..41376a0b0 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -30,6 +30,7 @@ import ( mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" ) +// Trains the MNIST dataset as a batch Job managed by MCAD, and asserts successful completion of the training job. func TestMNISTPyTorchMCAD(t *testing.T) { test := With(t) test.T().Parallel() diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index 4e804fd5b..202f3bcc7 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -32,6 +32,12 @@ import ( mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" ) +// Creates a Ray cluster, and trains the MNIST dataset using the CodeFlare SDK. +// Asserts successful completion of the training job. +// +// This covers the installation of the CodeFlare SDK, as well as the RBAC required +// for the SDK to successfully perform requests to the cluster, on behalf of the +// impersonated user. func TestMNISTRayClusterSDK(t *testing.T) { test := With(t) test.T().Parallel() diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index f20ba0a7a..b6b016538 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -31,6 +31,8 @@ import ( rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" ) +// Trains the MNIST dataset as a RayJob, executed by a Ray cluster managed by MCAD, +// and asserts successful completion of the training job. func TestMNISTRayJobMCADRayCluster(t *testing.T) { test := With(t) test.T().Parallel() From 99b8e7c6e9f86a37406291952d30f6a29c61a3ae Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 30 Jun 2023 17:11:26 +0200 Subject: [PATCH 065/100] test: Factorize e2e tests setup --- .github/workflows/e2e_tests.yaml | 55 +----------------------- Makefile | 6 +-- test/e2e/setup.sh | 73 ++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 56 deletions(-) create mode 100755 test/e2e/setup.sh diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 0c6bb2b91..435c72f0b 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -108,65 +108,14 @@ jobs: - name: Deploy CodeFlare stack id: deploy run: | - KUBERAY_VERSION=$(make get-kuberay-version) - echo Deploying KubeRay ${KUBERAY_VERSION} - kubectl create -k "github.com/ray-project/kuberay/ray-operator/config/default?ref=${KUBERAY_VERSION}&timeout=90s" - echo Deploying CodeFlare operator IMG="${REGISTRY_ADDRESS}"/codeflare-operator make image-push -e IMG="${IMG}" make deploy -e IMG="${IMG}" kubectl wait --timeout=120s --for=condition=Available=true deployment -n openshift-operators codeflare-operator-manager - echo Deploying MCAD controller - kubectl create ns codeflare-system - cat < Date: Fri, 30 Jun 2023 17:16:07 +0200 Subject: [PATCH 066/100] test: Update e2e tests local run documentation --- README.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 31f216405..31ce40f0b 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ CodeFlare Stack Compatibility Matrix The e2e tests can be executed locally by running the following commands: -1. Setup the test cluster: +1. Use an existing cluster, or set up a test cluster, e.g.: ```bash # Create a KinD cluster @@ -30,13 +30,20 @@ The e2e tests can be executed locally by running the following commands: $ make install ``` -2. Start the operator locally: +2. Set up the CodeFlare stack: + ```bash + $ make setup-e2e + ``` + +3. Start the operator locally: ```bash $ make run ``` -3. In a separate terminal, run the e2e suite: + Alternatively, You can run the operator from your IDE / debugger. + +4. In a separate terminal, run the e2e suite: ```bash $ make test-e2e From 8c1831ae50d2d56ad19e3d0b1c36b2b10c67b385 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 5 Jul 2023 14:30:52 +0200 Subject: [PATCH 067/100] test: Write logs also for jobs that have timed out --- test/e2e/mnist_pytorch_mcad_job_test.go | 6 +++--- test/e2e/mnist_raycluster_sdk_test.go | 6 +++--- test/e2e/mnist_rayjob_mcad_raycluster_test.go | 6 +++--- test/support/ray.go | 5 +++++ 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index 41376a0b0..168c7b134 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -145,6 +145,9 @@ func TestMNISTPyTorchMCAD(t *testing.T) { test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) + // Retrieving the job logs once it has completed or timed out + defer WriteJobLogs(test, job.Namespace, job.Name) + test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( Or( @@ -152,9 +155,6 @@ func TestMNISTPyTorchMCAD(t *testing.T) { WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), )) - // Retrieve the job logs - WriteJobLogs(test, job.Namespace, job.Name) - // Assert the job has completed successfully test.Expect(GetJob(test, job.Namespace, job.Name)). To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index 202f3bcc7..e8bb1f5c7 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -194,6 +194,9 @@ func TestMNISTRayClusterSDK(t *testing.T) { test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name) + // Retrieving the job logs once it has completed or timed out + defer WriteJobLogs(test, job.Namespace, job.Name) + test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( Or( @@ -201,9 +204,6 @@ func TestMNISTRayClusterSDK(t *testing.T) { WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), )) - // Retrieve the job logs - WriteJobLogs(test, job.Namespace, job.Name) - // Assert the job has completed successfully test.Expect(GetJob(test, job.Namespace, job.Name)). To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index b6b016538..958b30479 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -252,13 +252,13 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created RayJob %s/%s successfully", rayJob.Namespace, rayJob.Name) + // Retrieving the job logs once it has completed or timed out + defer WriteRayJobLogs(test, rayJob.Namespace, rayJob.Name) + test.T().Logf("Waiting for RayJob %s/%s to complete", rayJob.Namespace, rayJob.Name) test.Eventually(RayJob(test, rayJob.Namespace, rayJob.Name), TestTimeoutLong). Should(WithTransform(RayJobStatus, Satisfy(rayv1alpha1.IsJobTerminal))) - test.T().Logf("Retrieving RayJob %s/%s logs", rayJob.Namespace, rayJob.Name) - WriteToOutputDir(test, rayJob.Name, Log, GetRayJobLogs(test, rayJob.Namespace, rayJob.Name)) - // Assert the Ray job has completed successfully test.Expect(GetRayJob(test, rayJob.Namespace, rayJob.Name)). To(WithTransform(RayJobStatus, Equal(rayv1alpha1.JobStatusSucceeded))) diff --git a/test/support/ray.go b/test/support/ray.go index 3833d43b7..cd5a9d87d 100644 --- a/test/support/ray.go +++ b/test/support/ray.go @@ -63,3 +63,8 @@ func GetRayJobLogs(t Test, namespace, name string) []byte { return []byte(body["logs"]) } + +func WriteRayJobLogs(t Test, namespace, name string) { + t.T().Logf("Retrieving RayJob %s/%s logs", namespace, name) + WriteToOutputDir(t, name, Log, GetRayJobLogs(t, namespace, name)) +} From 2fa3f901d2f82efec476a85fa2a96ce8746f89dd Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 5 Jul 2023 16:35:07 +0200 Subject: [PATCH 068/100] doc: The operator should be started before setting up e2e tests --- README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 31ce40f0b..5984017e2 100644 --- a/README.md +++ b/README.md @@ -30,12 +30,7 @@ The e2e tests can be executed locally by running the following commands: $ make install ``` -2. Set up the CodeFlare stack: - ```bash - $ make setup-e2e - ``` - -3. Start the operator locally: +2. Start the operator locally: ```bash $ make run @@ -43,6 +38,12 @@ The e2e tests can be executed locally by running the following commands: Alternatively, You can run the operator from your IDE / debugger. +3. Set up the test CodeFlare stack: + + ```bash + $ make setup-e2e + ``` + 4. In a separate terminal, run the e2e suite: ```bash From e04fbfa87c852c2788b5be389fd52834279075a2 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 27 Jun 2023 10:09:48 +0200 Subject: [PATCH 069/100] fix: Update MCAD version when installing CRDs --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index b4844103f..ef3a05be2 100644 --- a/Makefile +++ b/Makefile @@ -242,7 +242,9 @@ endif .PHONY: install install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) fn run config/crd/mcad --image gcr.io/kpt-fn/apply-setters:v0.2.0 -- MCAD_CRD=$(MCAD_CRD) $(KUSTOMIZE) build config/crd | kubectl apply -f - + git restore config/* .PHONY: uninstall uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. From 92c616dcc3cc7c198303988cacb540aa75b6d807 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Fri, 7 Jul 2023 11:09:14 +0200 Subject: [PATCH 070/100] Use CodeFlare machine user to push PR into OpenShift community operators repository --- .github/workflows/tag-and-build.yml | 8 ++++---- Makefile | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 413ccf2d8..b823fc11b 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -94,15 +94,15 @@ jobs: - name: Build bundle and create PR in OpenShift community operators repository run: | - git config --global user.email "codeflare-ci@redhat.com" - git config --global user.name "CodeFlare CI" + git config --global user.email "138894154+codeflare-machine-account@users.noreply.github.com" + git config --global user.name "codeflare-machine-account" make openshift-community-operator-release env: VERSION: ${{ github.event.inputs.version }} PREVIOUS_VERSION: ${{ github.event.inputs.replaces }} INSTASCALE_VERSION: ${{ github.event.inputs.instascale-version }} MCAD_VERSION: ${{ github.event.inputs.mcad-version }} - GH_TOKEN: ${{ secrets.GH_PAT }} + GH_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} IMAGE_ORG_BASE: quay.io/${{ github.event.inputs.quay-organization }} OPERATORS_REPO_FORK_ORG: ${{ github.event.inputs.community-operators-prod-fork-organization }} OPERATORS_REPO_ORG: ${{ github.event.inputs.community-operators-prod-organization }} @@ -125,7 +125,7 @@ jobs: uses: stefanzweifel/git-auto-commit-action@v4 with: commit_message: Update dependency versions for release ${{ github.event.inputs.version }} - file_pattern: 'README.md controllers/defaults.go *.yaml *.tmpl Makefile' + file_pattern: 'README.md controllers/defaults.go *.yaml Makefile' - name: Creates a release in GitHub run: | diff --git a/Makefile b/Makefile index ef3a05be2..f1ff2de99 100644 --- a/Makefile +++ b/Makefile @@ -368,11 +368,11 @@ bundle-push: ## Push the bundle image. .PHONY: openshift-community-operator-release openshift-community-operator-release: install-gh-cli bundle ## build bundle and create PR in OpenShift community operators repository - git clone https://$(GH_TOKEN)@github.com/$(OPERATORS_REPO_FORK_ORG)/community-operators-prod.git + git clone https://x-access-token:$(GH_TOKEN)@github.com/$(OPERATORS_REPO_FORK_ORG)/community-operators-prod.git cd community-operators-prod && git remote add upstream https://github.com/$(OPERATORS_REPO_ORG)/community-operators-prod.git && git pull upstream main && git push origin main cp -r bundle community-operators-prod/operators/codeflare-operator/$(BUNDLE_VERSION) - cd community-operators-prod && git checkout -b codeflare-release-$(BUNDLE_VERSION) && git add operators/codeflare-operator/$(BUNDLE_VERSION)/* && git commit -m "add bundle manifests codeflare version $(BUNDLE_VERSION)" && git push origin codeflare-release-$(BUNDLE_VERSION) - gh pr create --repo $(OPERATORS_REPO_FORK_ORG)/community-operators-prod --title "CodeFlare $(BUNDLE_VERSION)" --body "New release of codeflare operator" --head $(OPERATORS_REPO_ORG):codeflare-release-$(BUNDLE_VERSION) --base main + cd community-operators-prod && git checkout -b codeflare-release-$(BUNDLE_VERSION) && git add operators/codeflare-operator/$(BUNDLE_VERSION)/* && git commit -m "add bundle manifests codeflare version $(BUNDLE_VERSION)" --signoff && git push origin codeflare-release-$(BUNDLE_VERSION) + gh pr create --repo $(OPERATORS_REPO_ORG)/community-operators-prod --title "CodeFlare $(BUNDLE_VERSION)" --body "New release of codeflare operator" --head $(OPERATORS_REPO_FORK_ORG):codeflare-release-$(BUNDLE_VERSION) --base main rm -rf community-operators-prod .PHONY: opm From 644d27bb9146049fa5e65b61bcb9507c473dcf1b Mon Sep 17 00:00:00 2001 From: anishasthana Date: Fri, 7 Jul 2023 16:00:26 +0000 Subject: [PATCH 071/100] Update dependency versions for release v0.0.5 --- Makefile | 4 ++-- README.md | 8 ++++---- controllers/defaults.go | 4 ++-- .../instascale_test_results/case_1/deployment.yaml | 2 +- .../instascale_test_results/case_2/deployment.yaml | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index f1ff2de99..c0037f0e6 100644 --- a/Makefile +++ b/Makefile @@ -12,10 +12,10 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # INSTASCALE_VERSION defines the default version of the InstaScale controller -INSTASCALE_VERSION ?= v0.0.4 +INSTASCALE_VERSION ?= v0.0.5 # MCAD_VERSION defines the default version of the MCAD controller -MCAD_VERSION ?= v1.31.0 +MCAD_VERSION ?= v1.32.0 # MCAD_REF, MCAD_REPO and MCAD_CRD define the reference to MCAD CRD resources MCAD_REF ?= release-${MCAD_VERSION} MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher diff --git a/README.md b/README.md index 5984017e2..cacc70196 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,10 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------| -| CodeFlare Operator | v0.0.4 | -| Multi-Cluster App Dispatcher | v1.31.0 | -| CodeFlare-SDK | v0.4.4 | -| InstaScale | v0.0.4 | +| CodeFlare Operator | v0.0.5 | +| Multi-Cluster App Dispatcher | v1.32.0 | +| CodeFlare-SDK | v0.5.0 | +| InstaScale | v0.0.5 | | KubeRay | v0.5.0 | diff --git a/controllers/defaults.go b/controllers/defaults.go index d3cac3d07..4dc0b777b 100644 --- a/controllers/defaults.go +++ b/controllers/defaults.go @@ -5,6 +5,6 @@ package controllers // *********************** const ( - MCADImage = "quay.io/project-codeflare/mcad-controller:release-v1.31.0" - InstaScaleImage = "quay.io/project-codeflare/instascale-controller:v0.0.4" + MCADImage = "quay.io/project-codeflare/mcad-controller:release-v1.32.0" + InstaScaleImage = "quay.io/project-codeflare/instascale-controller:v0.0.5" ) diff --git a/controllers/testdata/instascale_test_results/case_1/deployment.yaml b/controllers/testdata/instascale_test_results/case_1/deployment.yaml index 31c3896fb..bb398dbbd 100644 --- a/controllers/testdata/instascale_test_results/case_1/deployment.yaml +++ b/controllers/testdata/instascale_test_results/case_1/deployment.yaml @@ -19,7 +19,7 @@ spec: - name: instascale args: - "--configs-namespace=default" - image: quay.io/project-codeflare/instascale-controller:v0.0.4 + image: quay.io/project-codeflare/instascale-controller:v0.0.5 resources: limits: cpu: '2' diff --git a/controllers/testdata/instascale_test_results/case_2/deployment.yaml b/controllers/testdata/instascale_test_results/case_2/deployment.yaml index eb259ec86..ef7b8da11 100644 --- a/controllers/testdata/instascale_test_results/case_2/deployment.yaml +++ b/controllers/testdata/instascale_test_results/case_2/deployment.yaml @@ -19,7 +19,7 @@ spec: - name: instascale args: - "--configs-namespace=default" - image: quay.io/project-codeflare/instascale-controller:v0.0.4 + image: quay.io/project-codeflare/instascale-controller:v0.0.5 resources: limits: cpu: '1' From 598f32b1d03c7e3df8a5642d2ec311f83c4dc203 Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Sun, 9 Jul 2023 00:42:11 -0400 Subject: [PATCH 072/100] Fix mcad tag reference for CRDs Upstream is no longer publishing release branches of the form `release-vX.Y.Z` Instead, it is publishing tags of the form `vX.Y.Z`, this commit makes it so Signed-off-by: Anish Asthana --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c0037f0e6..5eb47e4c6 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,9 @@ MCAD_VERSION ?= v1.32.0 # MCAD_REF, MCAD_REPO and MCAD_CRD define the reference to MCAD CRD resources MCAD_REF ?= release-${MCAD_VERSION} MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher -MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_REF} +# Upstream MCAD is currently only creating release tags of the form `vX.Y.Z` (i.e the version) +# The image is still published using the MCAD_REF format (i.e release-vX.Y.Z) +MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_VERSION} # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) KUBERAY_VERSION ?= v0.5.0 From cd43ff56c4413984e17b64bae9ad142ee10be919 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Mon, 10 Jul 2023 13:21:00 +0200 Subject: [PATCH 073/100] Adjust Makefile versions before building operator image --- .github/workflows/tag-and-build.yml | 36 +++++++++++++++-------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index b823fc11b..5f07893ed 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -49,6 +49,9 @@ jobs: permissions: contents: write + env: + IMAGE_ORG_BASE: quay.io/${{ github.event.inputs.quay-organization }} + steps: - uses: actions/checkout@v3 @@ -73,6 +76,20 @@ jobs: - name: Install operator-sdk run: make install-operator-sdk + - name: Adjust Compatibility Matrix in readme + run: | + sed -i -E "s/(.*CodeFlare Operator.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.version }}\2/" README.md + sed -i -E "s/(.*Multi-Cluster App Dispatcher.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.mcad-version }}\2/" README.md + sed -i -E "s/(.*CodeFlare-SDK.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.codeflare-sdk-version }}\2/" README.md + sed -i -E "s/(.*InstaScale.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.instascale-version }}\2/" README.md + + - name: Adjust MCAD and InstaScale dependencies in the code + run: | + sed -i -E "s/(.*MCAD_VERSION \?= )v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.mcad-version }}\2/" Makefile + sed -i -E "s/(.*INSTASCALE_VERSION \?= )v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.instascale-version }}\2/" Makefile + sed -i -E "s/(.*instascale-controller:)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.instascale-version }}\2/" controllers/testdata/instascale_test_results/case_1/deployment.yaml + sed -i -E "s/(.*instascale-controller:)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.instascale-version }}\2/" controllers/testdata/instascale_test_results/case_2/deployment.yaml + - name: Login to Quay.io uses: redhat-actions/podman-login@v1 with: @@ -103,29 +120,14 @@ jobs: INSTASCALE_VERSION: ${{ github.event.inputs.instascale-version }} MCAD_VERSION: ${{ github.event.inputs.mcad-version }} GH_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} - IMAGE_ORG_BASE: quay.io/${{ github.event.inputs.quay-organization }} OPERATORS_REPO_FORK_ORG: ${{ github.event.inputs.community-operators-prod-fork-organization }} OPERATORS_REPO_ORG: ${{ github.event.inputs.community-operators-prod-organization }} - - name: Adjust Compatibility Matrix in readme - run: | - sed -i -E "s/(.*CodeFlare Operator.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.version }}\2/" README.md - sed -i -E "s/(.*Multi-Cluster App Dispatcher.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.mcad-version }}\2/" README.md - sed -i -E "s/(.*CodeFlare-SDK.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.codeflare-sdk-version }}\2/" README.md - sed -i -E "s/(.*InstaScale.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.instascale-version }}\2/" README.md - - - name: Adjust MCAD and InstaScale dependencies in the code - run: | - sed -i -E "s/(.*MCAD_VERSION \?= )v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.mcad-version }}\2/" Makefile - sed -i -E "s/(.*INSTASCALE_VERSION \?= )v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.instascale-version }}\2/" Makefile - sed -i -E "s/(.*instascale-controller:)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.instascale-version }}\2/" controllers/testdata/instascale_test_results/case_1/deployment.yaml - sed -i -E "s/(.*instascale-controller:)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.instascale-version }}\2/" controllers/testdata/instascale_test_results/case_2/deployment.yaml - - - name: Commit readme changes back to repository + - name: Commit changes in the code back to repository uses: stefanzweifel/git-auto-commit-action@v4 with: commit_message: Update dependency versions for release ${{ github.event.inputs.version }} - file_pattern: 'README.md controllers/defaults.go *.yaml Makefile' + file_pattern: 'README.md controllers/defaults.go *.yaml Makefile go.mod go.sum' - name: Creates a release in GitHub run: | From 1380bcc093874404c2950e737a0a9a447e6bb20a Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Mon, 10 Jul 2023 15:56:32 +0100 Subject: [PATCH 074/100] updating rbacs --- config/rbac/role.yaml | 7 +++++++ controllers/instascale_controller.go | 1 + 2 files changed, 8 insertions(+) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index ab9bc9054..d0c21ba62 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -96,6 +96,13 @@ rules: - get - patch - update +- apiGroups: + - config.openshift.io + resources: + - clusterversions + verbs: + - get + - list - apiGroups: - coordination.k8s.io resources: diff --git a/controllers/instascale_controller.go b/controllers/instascale_controller.go index 31fcb53b9..80e38a53d 100644 --- a/controllers/instascale_controller.go +++ b/controllers/instascale_controller.go @@ -90,6 +90,7 @@ func (r *InstaScaleReconciler) Apply(owner mf.Owner, params *InstaScaleParams, t // +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterroles;clusterrolebindings,verbs=get;list;watch;create;update;delete // +kubebuilder:rbac:groups=machine.openshift.io,resources=*,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=mcad.ibm.com,resources=appwrappers;queuejobs;schedulingspecs,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=config.openshift.io,resources=clusterversions,verbs=get;list // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. From d376f12c321cbd70ab9f3718656495f17cf3c717 Mon Sep 17 00:00:00 2001 From: anishasthana Date: Mon, 10 Jul 2023 16:37:28 +0000 Subject: [PATCH 075/100] Update dependency versions for release v0.0.6 --- README.md | 2 +- go.mod | 2 +- go.sum | 18 ++++++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cacc70196..e5f25443e 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------| -| CodeFlare Operator | v0.0.5 | +| CodeFlare Operator | v0.0.6 | | Multi-Cluster App Dispatcher | v1.32.0 | | CodeFlare-SDK | v0.5.0 | | InstaScale | v0.0.5 | diff --git a/go.mod b/go.mod index 621853fb6..7d8c01c8f 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/manifestival/manifestival v0.7.2 github.com/onsi/ginkgo/v2 v2.9.2 github.com/onsi/gomega v1.27.6 - github.com/project-codeflare/multi-cluster-app-dispatcher v1.31.0 + github.com/project-codeflare/multi-cluster-app-dispatcher v1.32.0 github.com/ray-project/kuberay/ray-operator v0.0.0-20230614221720-085c29d40fa9 go.uber.org/zap v1.24.0 k8s.io/api v0.26.3 diff --git a/go.sum b/go.sum index 8e877d1f6..f16a69965 100644 --- a/go.sum +++ b/go.sum @@ -33,6 +33,7 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v11.1.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= @@ -55,6 +56,7 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -79,6 +81,7 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -97,6 +100,7 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -313,9 +317,12 @@ github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:Fecb github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= @@ -330,6 +337,7 @@ github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -382,6 +390,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09 github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -400,6 +409,7 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -430,6 +440,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/project-codeflare/multi-cluster-app-dispatcher v1.31.0 h1:vq4fAuvlv4Zvnx0dA53WaYkTWG1BFxAkamxuzHfZO2M= github.com/project-codeflare/multi-cluster-app-dispatcher v1.31.0/go.mod h1:fmbU5LuV1Z2Sbu1FCEoVuw8qxDFcalXvkPyMfGZHHTc= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.32.0 h1:17lwqvxcWBTi9245lVZgJPvksGwAGA58s8dl7PzpbyE= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.32.0/go.mod h1:fmbU5LuV1Z2Sbu1FCEoVuw8qxDFcalXvkPyMfGZHHTc= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -471,7 +483,9 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= @@ -499,6 +513,7 @@ github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PK github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= @@ -511,7 +526,9 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -905,6 +922,7 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 70f0719289e5eed135827d65b3ea40af44121442 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Tue, 11 Jul 2023 09:46:23 +0200 Subject: [PATCH 076/100] Run go mod tidy before comitting files in release workflow --- .github/workflows/tag-and-build.yml | 4 ++++ go.sum | 18 ------------------ 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 5f07893ed..339d5b6fc 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -123,6 +123,10 @@ jobs: OPERATORS_REPO_FORK_ORG: ${{ github.event.inputs.community-operators-prod-fork-organization }} OPERATORS_REPO_ORG: ${{ github.event.inputs.community-operators-prod-organization }} + - name: Cleanup the go.mod and go.sum + run: | + go mod tidy + - name: Commit changes in the code back to repository uses: stefanzweifel/git-auto-commit-action@v4 with: diff --git a/go.sum b/go.sum index f16a69965..c143ff7a9 100644 --- a/go.sum +++ b/go.sum @@ -33,7 +33,6 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v11.1.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= @@ -56,7 +55,6 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -81,7 +79,6 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -100,7 +97,6 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -317,12 +313,9 @@ github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:Fecb github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= @@ -337,7 +330,6 @@ github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -390,7 +382,6 @@ github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09 github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -409,7 +400,6 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -438,8 +428,6 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.31.0 h1:vq4fAuvlv4Zvnx0dA53WaYkTWG1BFxAkamxuzHfZO2M= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.31.0/go.mod h1:fmbU5LuV1Z2Sbu1FCEoVuw8qxDFcalXvkPyMfGZHHTc= github.com/project-codeflare/multi-cluster-app-dispatcher v1.32.0 h1:17lwqvxcWBTi9245lVZgJPvksGwAGA58s8dl7PzpbyE= github.com/project-codeflare/multi-cluster-app-dispatcher v1.32.0/go.mod h1:fmbU5LuV1Z2Sbu1FCEoVuw8qxDFcalXvkPyMfGZHHTc= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -483,9 +471,7 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= @@ -513,7 +499,6 @@ github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PK github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= @@ -526,9 +511,7 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -922,7 +905,6 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 9c269d0881a0a37166c120963d747b543413df4e Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 15 Jun 2023 14:52:53 +0200 Subject: [PATCH 077/100] OLM install and upgrade PR check for CodeFlare stack --- .github/actions/kind/action.yml | 49 +++++ .../e2e => .github/resources-kind}/kind.yaml | 0 .../resources-olm-upgrade/catalogsource.yaml | 12 ++ .../resources-olm-upgrade/operatorgroup.yaml | 5 + .../resources-olm-upgrade/subscription.yaml | 11 + .github/workflows/e2e_tests.yaml | 46 +---- .github/workflows/olm_tests.yaml | 190 ++++++++++++++++++ Makefile | 37 +++- 8 files changed, 305 insertions(+), 45 deletions(-) create mode 100644 .github/actions/kind/action.yml rename {test/e2e => .github/resources-kind}/kind.yaml (100%) create mode 100644 .github/resources-olm-upgrade/catalogsource.yaml create mode 100644 .github/resources-olm-upgrade/operatorgroup.yaml create mode 100644 .github/resources-olm-upgrade/subscription.yaml create mode 100644 .github/workflows/olm_tests.yaml diff --git a/.github/actions/kind/action.yml b/.github/actions/kind/action.yml new file mode 100644 index 000000000..58122e8a2 --- /dev/null +++ b/.github/actions/kind/action.yml @@ -0,0 +1,49 @@ +name: "Set up KinD" +description: "Step to start and configure KinD cluster" + +runs: + using: "composite" + steps: + - name: Init directories + shell: bash + run: | + TEMP_DIR="$(pwd)/tmp" + mkdir -p "${TEMP_DIR}" + echo "TEMP_DIR=${TEMP_DIR}" >> $GITHUB_ENV + + mkdir -p "$(pwd)/bin" + echo "$(pwd)/bin" >> $GITHUB_PATH + + - name: Container image registry + shell: bash + run: | + podman run -d -p 5000:5000 --name registry registry:2.8.1 + + export REGISTRY_ADDRESS=$(hostname -i):5000 + echo "REGISTRY_ADDRESS=${REGISTRY_ADDRESS}" >> $GITHUB_ENV + echo "Container image registry started at ${REGISTRY_ADDRESS}" + + KIND_CONFIG_FILE=${{ env.TEMP_DIR }}/kind.yaml + echo "KIND_CONFIG_FILE=${KIND_CONFIG_FILE}" >> $GITHUB_ENV + envsubst < .github/resources-kind/kind.yaml > ${KIND_CONFIG_FILE} + + sudo --preserve-env=REGISTRY_ADDRESS sh -c 'cat > /etc/containers/registries.conf.d/local.conf <> $GITHUB_ENV - - mkdir -p "$(pwd)/bin" - echo "$(pwd)/bin" >> $GITHUB_PATH - - name: Set Go uses: actions/setup-go@v3 with: @@ -73,37 +64,8 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} - - name: Container image registry - run: | - podman run -d -p 5000:5000 --name registry registry:2.8.1 - - export REGISTRY_ADDRESS=$(hostname -i):5000 - echo "REGISTRY_ADDRESS=${REGISTRY_ADDRESS}" >> $GITHUB_ENV - echo "Container image registry started at ${REGISTRY_ADDRESS}" - - KIND_CONFIG_FILE=${{ env.TEMP_DIR }}/kind.yaml - echo "KIND_CONFIG_FILE=${KIND_CONFIG_FILE}" >> $GITHUB_ENV - envsubst < ./test/e2e/kind.yaml > ${KIND_CONFIG_FILE} - - sudo --preserve-env=REGISTRY_ADDRESS sh -c 'cat > /etc/containers/registries.conf.d/local.conf <> $GITHUB_ENV diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml new file mode 100644 index 000000000..9e1061c91 --- /dev/null +++ b/.github/workflows/olm_tests.yaml @@ -0,0 +1,190 @@ +# This workflow will build the CodeFlare Operator image and catalog containing bundle with this image, execute OLM upgrade tests using this catalog + +name: OLM Install and Upgrade + +on: + pull_request: + branches: + - main + - 'release-*' + paths-ignore: + - 'docs/**' + - '**.adoc' + - '**.md' + - 'LICENSE' + +concurrency: + group: ${{ github.head_ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + kubernetes: + runs-on: ubuntu-20.04 + timeout-minutes: 60 + env: + OLM_VERSION: v0.24.0 + VERSION: "v0.0.0-ghaction" # Need to supply some semver version for bundle to be properly generated + CATALOG_BASE_IMG: "registry.access.redhat.com/redhat/community-operator-index:v4.13" + CODEFLARE_TEST_TIMEOUT_SHORT: "1m" + CODEFLARE_TEST_TIMEOUT_MEDIUM: "5m" + CODEFLARE_TEST_TIMEOUT_LONG: "10m" + + steps: + - name: Cleanup + run: | + ls -lart + echo "Initial status:" + df -h + + echo "Cleaning up resources:" + sudo swapoff -a + sudo rm -f /swapfile + sudo apt clean + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + docker rmi $(docker image ls -aq) + + echo "Final status:" + df -h + + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # fetching also previous commits to get tags + + - name: Set Go + uses: actions/setup-go@v3 + with: + go-version: v1.18 + + - name: Set up gotestfmt + uses: gotesttools/gotestfmt-action@v2 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup and start KinD cluster + uses: ./.github/actions/kind + + - name: Install OLM + run: | + kubectl create -f https://github.com/operator-framework/operator-lifecycle-manager/releases/download/${OLM_VERSION}/crds.yaml + # wait for a while to be sure CRDs are installed + sleep 1 + kubectl create -f https://github.com/operator-framework/operator-lifecycle-manager/releases/download/${OLM_VERSION}/olm.yaml + + - name: Create openshift-operator namespace and OperatorGroup + run: | + # Need to use openshift-operator namespace due to https://github.com/project-codeflare/codeflare-operator/issues/161 + kubectl create namespace openshift-operators + kubectl create -f .github/resources-olm-upgrade/operatorgroup.yaml + + - name: Deploy latest released CodeFlare operator from OLM + id: deploy + run: | + echo Deploying CodeFlare operator using Subscription + envsubst < .github/resources-olm-upgrade/catalogsource.yaml > ${{ env.TEMP_DIR }}/catalogsource.yaml + envsubst < .github/resources-olm-upgrade/subscription.yaml > ${{ env.TEMP_DIR }}/subscription.yaml + + kubectl create -f ${{ env.TEMP_DIR }}/catalogsource.yaml + make wait-for-catalog-source + + kubectl create -f ${{ env.TEMP_DIR }}/subscription.yaml + + echo Waiting for Subscription to be ready + make wait-for-subscription + + echo Waiting for Deployment to be ready + make wait-for-deployment -e TIMEOUT=60 -e DEPLOYMENT_NAME="codeflare-operator-manager" -e DEPLOYMENT_NAMESPACE="openshift-operators" + env: + CATALOG_SOURCE_NAME: "codeflare-olm-test" + CATALOG_SOURCE_NAMESPACE: "olm" + SUBSCRIPTION_NAME: "codeflare-operator" + SUBSCRIPTION_NAMESPACE: "openshift-operators" + + - name: Store latest CSV version as PREVIOUS_VERSION env variable (used for bundle build) + run: | + CSV_VERSION=$(kubectl get ClusterServiceVersion -l operators.coreos.com/codeflare-operator.openshift-operators='' -n openshift-operators -o json | jq -r .items[].spec.version) + echo "PREVIOUS_VERSION=v$CSV_VERSION" >> $GITHUB_ENV + + - name: Deploy CodeFlare stack (MCAD, KubeRay) + run: | + make setup-e2e + + - name: Build operator and catalog image + run: | + make image-push + make bundle-build + make bundle-push + make catalog-build-from-index + make catalog-push + env: + IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator:v0.0.1" + BUNDLE_IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator-bundle:v0.0.1" + CATALOG_IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator-catalog:v0.0.1" + OPM_BUNDLE_OPT: "--use-http" + BUNDLE_PUSH_OPT: "--tls-verify=false" + CATALOG_PUSH_OPT: "--tls-verify=false" + + - name: Update Operator to the built version + run: | + ORIGINAL_POD_NAME=$(kubectl get pod -l app.kubernetes.io/name=codeflare-operator -n openshift-operators -o json | jq -r .items[].metadata.name) + echo "Running old operator pod name is ${ORIGINAL_POD_NAME}" + + echo Updating custom CatalogSource image to the built CatalogSource with latest operator + kubectl patch CatalogSource codeflare-olm-test -n olm --type merge --patch "{\"spec\":{\"image\":\"${CATALOG_IMG}\"}}" + + echo Waiting for previous operator pod to get deleted + kubectl wait --timeout=120s --for=delete pod/${ORIGINAL_POD_NAME} -n openshift-operators + + echo Waiting for Subscription to be ready + make wait-for-subscription + + echo Waiting for Deployment to be ready + make wait-for-deployment -e TIMEOUT=60 -e DEPLOYMENT_NAME="codeflare-operator-manager" -e DEPLOYMENT_NAMESPACE="openshift-operators" + + echo Checking that correct CSV is available + CSV_VERSION=$(kubectl get ClusterServiceVersion/codeflare-operator.${VERSION} -n openshift-operators -o json | jq -r .spec.version) + if [ "v${CSV_VERSION}" != "${VERSION}" ]; then + echo "CSV version v${CSV_VERSION} doesn't match expected version ${VERSION}" + exit 1 + fi + env: + CATALOG_IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator-catalog:v0.0.1" + SUBSCRIPTION_NAME: "codeflare-operator" + SUBSCRIPTION_NAMESPACE: "openshift-operators" + + - name: Run e2e tests against built operator + run: | + export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} + echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV + + set -euo pipefail + go test -timeout 30m -v ./test/e2e -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt + + - name: Print CodeFlare operator logs + if: always() && steps.deploy.outcome == 'success' + run: | + echo "Printing CodeFlare operator logs" + kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator | tee ${CODEFLARE_TEST_OUTPUT_DIR}/codeflare-operator.log + + - name: Print MCAD controller logs + if: always() && steps.deploy.outcome == 'success' + run: | + echo "Printing MCAD controller logs" + kubectl logs -n codeflare-system --tail -1 -l component=multi-cluster-application-dispatcher | tee ${CODEFLARE_TEST_OUTPUT_DIR}/mcad.log + + - name: Print KubeRay operator logs + if: always() && steps.deploy.outcome == 'success' + run: | + echo "Printing KubeRay operator logs" + kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kuberay.log + + - name: Upload logs + uses: actions/upload-artifact@v3 + if: always() && steps.deploy.outcome == 'success' + with: + name: logs + retention-days: 10 + path: | + ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/**/*.log diff --git a/Makefile b/Makefile index 5eb47e4c6..f10a87350 100644 --- a/Makefile +++ b/Makefile @@ -342,7 +342,7 @@ OPERATOR_SDK_DL_URL := https://github.com/operator-framework/operator-sdk/releas .PHONY: install-operator-sdk install-operator-sdk: $(OPERATOR_SDK) ## Download fixed version operator-sdk binary for consist outcome $(OPERATOR_SDK): $(LOCALBIN) - curl -L $(OPERATOR_SDK_DL_URL)/operator-sdk_$(shell go env GOOS)_$(shell go env GOARCH) --output-dir $(LOCALBIN) --output operator-sdk + curl -L $(OPERATOR_SDK_DL_URL)/operator-sdk_$(shell go env GOOS)_$(shell go env GOARCH) --output $(LOCALBIN)/operator-sdk chmod +x $(OPERATOR_SDK) .PHONY: validate-bundle @@ -366,7 +366,7 @@ bundle-build: bundle ## Build the bundle image. .PHONY: bundle-push bundle-push: ## Push the bundle image. - $(MAKE) image-push IMG=$(BUNDLE_IMG) + podman push $(BUNDLE_IMG) $(BUNDLE_PUSH_OPT) .PHONY: openshift-community-operator-release openshift-community-operator-release: install-gh-cli bundle ## build bundle and create PR in OpenShift community operators repository @@ -413,10 +413,21 @@ endif catalog-build: opm ## Build a catalog image. $(OPM) index add --container-tool podman --mode semver --tag $(CATALOG_IMG) --bundles $(BUNDLE_IMGS) $(FROM_INDEX_OPT) +# Build a catalog image by adding bundle images to existing catalog using the operator package manager tool, 'opm'. +.PHONY: catalog-build-from-index +catalog-build-from-index: opm ## Build a catalog image. + mkdir catalog + $(OPM) render $(CATALOG_BASE_IMG) -o yaml > catalog/bundles.yaml + $(OPM) render $(BUNDLE_IMG) $(OPM_BUNDLE_OPT) > catalog/codeflare-operator-bundle.yaml + sed -i -E "s/(.*)(- name: codeflare-operator.$(PREVIOUS_VERSION).*)/\1- name: codeflare-operator.$(VERSION)\n replaces: codeflare-operator.$(PREVIOUS_VERSION)\n\2/" catalog/bundles.yaml + $(OPM) validate catalog + $(OPM) generate dockerfile catalog + podman build . -f catalog.Dockerfile -t $(CATALOG_IMG) + # Push the catalog image. .PHONY: catalog-push catalog-push: ## Push a catalog image. - $(MAKE) image-push IMG=$(CATALOG_IMG) + podman push $(CATALOG_IMG) $(CATALOG_PUSH_OPT) .PHONY: test-unit test-unit: defaults manifests generate fmt vet envtest ## Run unit tests. @@ -429,3 +440,23 @@ test-e2e: defaults manifests generate fmt vet ## Run e2e tests. .PHONY: setup-e2e setup-e2e: ## Set up e2e tests. KUBERAY_VERSION=$(KUBERAY_VERSION) test/e2e/setup.sh + +# Wait until CatalogSource is in "READY" state +.PHONY: wait-for-catalog-source +wait-for-catalog-source: + timeout 120 bash -c 'while [[ "$$(kubectl get catalogsource/'$(CATALOG_SOURCE_NAME)' -n '$(CATALOG_SOURCE_NAMESPACE)' -o json | jq -r .status.connectionState.lastObservedState)" != "READY" ]]; do sleep 5 && echo "$$(kubectl get catalogsource/'$(CATALOG_SOURCE_NAME)' -n '$(CATALOG_SOURCE_NAMESPACE)' -o json | jq -r .status.connectionState.lastObservedState)" ; done' + +# Wait until Subscription is in "AtLatestKnown" state +.PHONY: wait-for-subscription +wait-for-subscription: + timeout 300 bash -c 'while [[ "$$(kubectl get subscription/'$(SUBSCRIPTION_NAME)' -n '$(SUBSCRIPTION_NAMESPACE)' -o json | jq -r .status.state)" != "AtLatestKnown" ]]; do sleep 5 && echo "$$(kubectl get subscription/'$(SUBSCRIPTION_NAME)' -n '$(SUBSCRIPTION_NAMESPACE)' -o json | jq -r .status.state)" ; done' + +# Default timeout for waiting actions +TIMEOUT ?= 180 + +# Wait until Deployment is in "Available" state +.PHONY: wait-for-deployment +wait-for-deployment: + # Wait until Deployment exists first, then use kubectl wait + timeout $(TIMEOUT) bash -c 'until [[ $$(kubectl get deployment/'$(DEPLOYMENT_NAME)' -n '$(DEPLOYMENT_NAMESPACE)') ]]; do sleep 5 && echo "$$(kubectl get deployment/'$(DEPLOYMENT_NAME)' -n '$(DEPLOYMENT_NAMESPACE)')"; done' + kubectl wait --timeout=$(TIMEOUT)s --for=condition=Available=true deployment/$(DEPLOYMENT_NAME) -n $(DEPLOYMENT_NAMESPACE) From 0a84b1d7f027fbe431532242a296d4b106b49623 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Tue, 18 Jul 2023 15:50:52 +0100 Subject: [PATCH 078/100] Expose CodeFlare Operator metrics endpoint --- config/default/manager_auth_proxy_patch.yaml | 2 +- config/rbac/auth_proxy_service.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml index 24d5b1194..31e1a3648 100644 --- a/config/default/manager_auth_proxy_patch.yaml +++ b/config/default/manager_auth_proxy_patch.yaml @@ -35,5 +35,5 @@ spec: - name: manager args: - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=127.0.0.1:8080" + - "--metrics-bind-address=0.0.0.0:8080" - "--leader-elect" diff --git a/config/rbac/auth_proxy_service.yaml b/config/rbac/auth_proxy_service.yaml index 250100420..c53c92d18 100644 --- a/config/rbac/auth_proxy_service.yaml +++ b/config/rbac/auth_proxy_service.yaml @@ -8,7 +8,7 @@ spec: - name: https port: 8443 protocol: TCP - targetPort: https + targetPort: 8080 selector: app.kubernetes.io/name: codeflare-operator app.kubernetes.io/part-of: codeflare From 1ff11205f5f3654a8abb5f0694a812fe1ef18d37 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Mon, 24 Jul 2023 13:25:53 +0200 Subject: [PATCH 079/100] Add Ray cluster support in test support package --- test/support/conditions.go | 15 ++++++++++++++- test/support/ray.go | 17 +++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/test/support/conditions.go b/test/support/conditions.go index e7c5097ad..16b26583e 100644 --- a/test/support/conditions.go +++ b/test/support/conditions.go @@ -18,6 +18,7 @@ limitations under the License. package support import ( + appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" ) @@ -34,7 +35,10 @@ func ConditionStatus[T conditionType](conditionType T) func(any) corev1.Conditio if c := getJobCondition(o.Status.Conditions, batchv1.JobConditionType(conditionType)); c != nil { return c.Status } - + case *appsv1.Deployment: + if c := getDeploymentCondition(o.Status.Conditions, appsv1.DeploymentConditionType(conditionType)); c != nil { + return c.Status + } } return corev1.ConditionUnknown @@ -51,3 +55,12 @@ func getJobCondition(conditions []batchv1.JobCondition, conditionType batchv1.Jo } return nil } + +func getDeploymentCondition(conditions []appsv1.DeploymentCondition, conditionType appsv1.DeploymentConditionType) *appsv1.DeploymentCondition { + for _, c := range conditions { + if c.Type == conditionType { + return &c + } + } + return nil +} diff --git a/test/support/ray.go b/test/support/ray.go index cd5a9d87d..52bf2d651 100644 --- a/test/support/ray.go +++ b/test/support/ray.go @@ -68,3 +68,20 @@ func WriteRayJobLogs(t Test, namespace, name string) { t.T().Logf("Retrieving RayJob %s/%s logs", namespace, name) WriteToOutputDir(t, name, Log, GetRayJobLogs(t, namespace, name)) } + +func RayCluster(t Test, namespace, name string) func(g gomega.Gomega) *rayv1alpha1.RayCluster { + return func(g gomega.Gomega) *rayv1alpha1.RayCluster { + cluster, err := t.Client().Ray().RayV1alpha1().RayClusters(namespace).Get(t.Ctx(), name, metav1.GetOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + return cluster + } +} + +func GetRayCluster(t Test, namespace, name string) *rayv1alpha1.RayCluster { + t.T().Helper() + return RayCluster(t, namespace, name)(t) +} + +func RayClusterState(cluster *rayv1alpha1.RayCluster) rayv1alpha1.ClusterState { + return cluster.Status.State +} From c885e0aeaba81922dfe65a45f648f59a81da50bd Mon Sep 17 00:00:00 2001 From: Wen Zhou Date: Sun, 23 Jul 2023 18:07:50 +0200 Subject: [PATCH 080/100] Update: remove duplicated resource and should have lower in order being depend on others --- controllers/mcad_controller.go | 1 - 1 file changed, 1 deletion(-) diff --git a/controllers/mcad_controller.go b/controllers/mcad_controller.go index f21023cdb..01f83060d 100644 --- a/controllers/mcad_controller.go +++ b/controllers/mcad_controller.go @@ -224,7 +224,6 @@ func (r *MCADReconciler) SetupWithManager(mgr ctrl.Manager) error { }) return ctrl.NewControllerManagedBy(mgr). For(&v1alpha1.MCAD{}). - Owns(&appsv1.Deployment{}). Owns(&corev1.ConfigMap{}). Owns(&corev1.Service{}). Owns(&corev1.ServiceAccount{}). From c9929bd16bb1d98f8b3853e56aa310c4168eb1e3 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Tue, 25 Jul 2023 09:21:04 +0200 Subject: [PATCH 081/100] Remove waiting loops for kubernetes resources from Makefile --- .github/actions/kind/action.yml | 2 +- .github/workflows/olm_tests.yaml | 19 ++++++++++++++----- Makefile | 20 -------------------- 3 files changed, 15 insertions(+), 26 deletions(-) diff --git a/.github/actions/kind/action.yml b/.github/actions/kind/action.yml index 58122e8a2..268c48a51 100644 --- a/.github/actions/kind/action.yml +++ b/.github/actions/kind/action.yml @@ -35,7 +35,7 @@ runs: EOF' - name: Setup KinD cluster - uses: helm/kind-action@v1.5.0 + uses: helm/kind-action@v1.8.0 with: cluster_name: cluster version: v0.17.0 diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index 9e1061c91..6f88029ee 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -72,6 +72,11 @@ jobs: # wait for a while to be sure CRDs are installed sleep 1 kubectl create -f https://github.com/operator-framework/operator-lifecycle-manager/releases/download/${OLM_VERSION}/olm.yaml + echo Wait for default CatalogSource to start + kubectl wait -n ${{ env.CATALOG_SOURCE_NAMESPACE }} catalogsource/${{ env.CATALOG_SOURCE_NAME }} --for=jsonpath='{.status.connectionState.lastObservedState}'=READY --timeout=180s + env: + CATALOG_SOURCE_NAME: "operatorhubio-catalog" + CATALOG_SOURCE_NAMESPACE: "olm" - name: Create openshift-operator namespace and OperatorGroup run: | @@ -87,15 +92,18 @@ jobs: envsubst < .github/resources-olm-upgrade/subscription.yaml > ${{ env.TEMP_DIR }}/subscription.yaml kubectl create -f ${{ env.TEMP_DIR }}/catalogsource.yaml - make wait-for-catalog-source + + echo Wait for CatalogSource ${{ env.CATALOG_SOURCE_NAME }} to start + kubectl wait -n ${{ env.CATALOG_SOURCE_NAMESPACE }} catalogsource/${{ env.CATALOG_SOURCE_NAME }} --for=jsonpath='{.status.connectionState.lastObservedState}'=READY --timeout=180s kubectl create -f ${{ env.TEMP_DIR }}/subscription.yaml echo Waiting for Subscription to be ready - make wait-for-subscription + kubectl wait -n ${{ env.SUBSCRIPTION_NAMESPACE }} subscription/${{ env.SUBSCRIPTION_NAME }} --for=jsonpath='{.status.state}'=AtLatestKnown --timeout=180s echo Waiting for Deployment to be ready - make wait-for-deployment -e TIMEOUT=60 -e DEPLOYMENT_NAME="codeflare-operator-manager" -e DEPLOYMENT_NAMESPACE="openshift-operators" + timeout 60 bash -c 'until [[ $(kubectl get deployment/codeflare-operator-manager -n '${{ env.SUBSCRIPTION_NAMESPACE }}') ]]; do sleep 5 && echo "$(kubectl get deployment/codeflare-operator-manager -n '${{ env.SUBSCRIPTION_NAMESPACE }}')"; done' + kubectl wait -n ${{ env.SUBSCRIPTION_NAMESPACE }} deployment/codeflare-operator-manager --for=condition=Available=true --timeout=60s env: CATALOG_SOURCE_NAME: "codeflare-olm-test" CATALOG_SOURCE_NAMESPACE: "olm" @@ -138,10 +146,11 @@ jobs: kubectl wait --timeout=120s --for=delete pod/${ORIGINAL_POD_NAME} -n openshift-operators echo Waiting for Subscription to be ready - make wait-for-subscription + kubectl wait -n ${{ env.SUBSCRIPTION_NAMESPACE }} subscription/${{ env.SUBSCRIPTION_NAME }} --for=jsonpath='{.status.state}'=AtLatestKnown --timeout=180s echo Waiting for Deployment to be ready - make wait-for-deployment -e TIMEOUT=60 -e DEPLOYMENT_NAME="codeflare-operator-manager" -e DEPLOYMENT_NAMESPACE="openshift-operators" + timeout 60 bash -c 'until [[ $(kubectl get deployment/codeflare-operator-manager -n '${{ env.SUBSCRIPTION_NAMESPACE }}') ]]; do sleep 5 && echo "$(kubectl get deployment/codeflare-operator-manager -n '${{ env.SUBSCRIPTION_NAMESPACE }}')"; done' + kubectl wait -n ${{ env.SUBSCRIPTION_NAMESPACE }} deployment/codeflare-operator-manager --for=condition=Available=true --timeout=60s echo Checking that correct CSV is available CSV_VERSION=$(kubectl get ClusterServiceVersion/codeflare-operator.${VERSION} -n openshift-operators -o json | jq -r .spec.version) diff --git a/Makefile b/Makefile index f10a87350..4aea94539 100644 --- a/Makefile +++ b/Makefile @@ -440,23 +440,3 @@ test-e2e: defaults manifests generate fmt vet ## Run e2e tests. .PHONY: setup-e2e setup-e2e: ## Set up e2e tests. KUBERAY_VERSION=$(KUBERAY_VERSION) test/e2e/setup.sh - -# Wait until CatalogSource is in "READY" state -.PHONY: wait-for-catalog-source -wait-for-catalog-source: - timeout 120 bash -c 'while [[ "$$(kubectl get catalogsource/'$(CATALOG_SOURCE_NAME)' -n '$(CATALOG_SOURCE_NAMESPACE)' -o json | jq -r .status.connectionState.lastObservedState)" != "READY" ]]; do sleep 5 && echo "$$(kubectl get catalogsource/'$(CATALOG_SOURCE_NAME)' -n '$(CATALOG_SOURCE_NAMESPACE)' -o json | jq -r .status.connectionState.lastObservedState)" ; done' - -# Wait until Subscription is in "AtLatestKnown" state -.PHONY: wait-for-subscription -wait-for-subscription: - timeout 300 bash -c 'while [[ "$$(kubectl get subscription/'$(SUBSCRIPTION_NAME)' -n '$(SUBSCRIPTION_NAMESPACE)' -o json | jq -r .status.state)" != "AtLatestKnown" ]]; do sleep 5 && echo "$$(kubectl get subscription/'$(SUBSCRIPTION_NAME)' -n '$(SUBSCRIPTION_NAMESPACE)' -o json | jq -r .status.state)" ; done' - -# Default timeout for waiting actions -TIMEOUT ?= 180 - -# Wait until Deployment is in "Available" state -.PHONY: wait-for-deployment -wait-for-deployment: - # Wait until Deployment exists first, then use kubectl wait - timeout $(TIMEOUT) bash -c 'until [[ $$(kubectl get deployment/'$(DEPLOYMENT_NAME)' -n '$(DEPLOYMENT_NAMESPACE)') ]]; do sleep 5 && echo "$$(kubectl get deployment/'$(DEPLOYMENT_NAME)' -n '$(DEPLOYMENT_NAMESPACE)')"; done' - kubectl wait --timeout=$(TIMEOUT)s --for=condition=Available=true deployment/$(DEPLOYMENT_NAME) -n $(DEPLOYMENT_NAMESPACE) From a82274750112f5d01b18f3de029549a720f12320 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 27 Jul 2023 13:46:17 +0200 Subject: [PATCH 082/100] Add Ray cluster REST API support in test support package --- go.mod | 6 +- go.sum | 10 ++- test/support/client.go | 14 ++++ test/support/conditions.go | 15 ++++ test/support/mcad.go | 3 +- test/support/ray.go | 3 +- test/support/ray_api.go | 45 +++++++++++ test/support/ray_cluster_client.go | 115 +++++++++++++++++++++++++++++ test/support/route.go | 33 +++++++++ 9 files changed, 236 insertions(+), 8 deletions(-) create mode 100644 test/support/ray_api.go create mode 100644 test/support/ray_cluster_client.go create mode 100644 test/support/route.go diff --git a/go.mod b/go.mod index 7d8c01c8f..8cbc5ec9b 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,8 @@ require ( github.com/manifestival/manifestival v0.7.2 github.com/onsi/ginkgo/v2 v2.9.2 github.com/onsi/gomega v1.27.6 + github.com/openshift/api v0.0.0-20230213134911-7ba313770556 + github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c github.com/project-codeflare/multi-cluster-app-dispatcher v1.32.0 github.com/ray-project/kuberay/ray-operator v0.0.0-20230614221720-085c29d40fa9 go.uber.org/zap v1.24.0 @@ -36,7 +38,7 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/go-cmp v0.5.9 // indirect - github.com/google/gofuzz v1.1.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect github.com/google/uuid v1.1.2 // indirect github.com/imdario/mergo v0.3.12 // indirect @@ -56,7 +58,7 @@ require ( go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect golang.org/x/net v0.8.0 // indirect - golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect + golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect golang.org/x/sys v0.6.0 // indirect golang.org/x/term v0.6.0 // indirect golang.org/x/text v0.8.0 // indirect diff --git a/go.sum b/go.sum index c143ff7a9..815e362bb 100644 --- a/go.sum +++ b/go.sum @@ -280,8 +280,9 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -418,6 +419,10 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= +github.com/openshift/api v0.0.0-20230213134911-7ba313770556 h1:7W2fOhJicyEff24VaF7ASNzPtYvr+iSCVft4SIBAzaE= +github.com/openshift/api v0.0.0-20230213134911-7ba313770556/go.mod h1:aQ6LDasvHMvHZXqLHnX2GRmnfTWCF/iIwz8EMTTIE9A= +github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c h1:CV76yFOTXmq9VciBR3Bve5ZWzSxdft7gaMVB3kS0rwg= +github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c/go.mod h1:lFMO8mLHXWFzSdYvGNo8ivF9SfF6zInA8ZGw4phRnUE= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= @@ -635,8 +640,9 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b h1:clP8eMhB30EHdc0bd2Twtq6kgU7yl5ub2cQLSdrv1Dg= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 h1:OSnWWcOd/CtWQC2cYSBgbTSJv3ciqd8r54ySIW2y3RE= +golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/test/support/client.go b/test/support/client.go index a8efa76d2..73931d645 100644 --- a/test/support/client.go +++ b/test/support/client.go @@ -24,6 +24,8 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" + routev1 "github.com/openshift/client-go/route/clientset/versioned" + codeflareclient "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" mcadclient "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/client/clientset/controller-versioned" rayclient "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned" @@ -31,6 +33,7 @@ import ( type Client interface { Core() kubernetes.Interface + Route() routev1.Interface CodeFlare() codeflareclient.Interface MCAD() mcadclient.Interface Ray() rayclient.Interface @@ -38,6 +41,7 @@ type Client interface { type testClient struct { core kubernetes.Interface + route routev1.Interface codeflare codeflareclient.Interface mcad mcadclient.Interface ray rayclient.Interface @@ -49,6 +53,10 @@ func (t *testClient) Core() kubernetes.Interface { return t.core } +func (t *testClient) Route() routev1.Interface { + return t.route +} + func (t *testClient) CodeFlare() codeflareclient.Interface { return t.codeflare } @@ -75,6 +83,11 @@ func newTestClient() (Client, error) { return nil, err } + routeClient, err := routev1.NewForConfig(cfg) + if err != nil { + return nil, err + } + codeFlareClient, err := codeflareclient.NewForConfig(cfg) if err != nil { return nil, err @@ -92,6 +105,7 @@ func newTestClient() (Client, error) { return &testClient{ core: kubeClient, + route: routeClient, codeflare: codeFlareClient, mcad: mcadClient, ray: rayClient, diff --git a/test/support/conditions.go b/test/support/conditions.go index 16b26583e..70d0ebe9c 100644 --- a/test/support/conditions.go +++ b/test/support/conditions.go @@ -21,6 +21,8 @@ import ( appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + + routev1 "github.com/openshift/api/route/v1" ) type conditionType interface { @@ -39,6 +41,10 @@ func ConditionStatus[T conditionType](conditionType T) func(any) corev1.Conditio if c := getDeploymentCondition(o.Status.Conditions, appsv1.DeploymentConditionType(conditionType)); c != nil { return c.Status } + case *routev1.Route: + if c := getRouteCondition(o.Status.Ingress[0].Conditions, routev1.RouteIngressConditionType(conditionType)); c != nil { + return c.Status + } } return corev1.ConditionUnknown @@ -64,3 +70,12 @@ func getDeploymentCondition(conditions []appsv1.DeploymentCondition, conditionTy } return nil } + +func getRouteCondition(conditions []routev1.RouteIngressCondition, conditionType routev1.RouteIngressConditionType) *routev1.RouteIngressCondition { + for _, c := range conditions { + if c.Type == conditionType { + return &c + } + } + return nil +} diff --git a/test/support/mcad.go b/test/support/mcad.go index 4f268985f..607a94f74 100644 --- a/test/support/mcad.go +++ b/test/support/mcad.go @@ -18,11 +18,10 @@ package support import ( "github.com/onsi/gomega" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" ) func AppWrapper(t Test, namespace *corev1.Namespace, name string) func(g gomega.Gomega) *mcadv1beta1.AppWrapper { diff --git a/test/support/ray.go b/test/support/ray.go index 52bf2d651..4dc6ebc1b 100644 --- a/test/support/ray.go +++ b/test/support/ray.go @@ -20,10 +20,9 @@ import ( "encoding/json" "github.com/onsi/gomega" + rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" ) const RayJobDefaultClusterSelectorKey = "ray.io/cluster" diff --git a/test/support/ray_api.go b/test/support/ray_api.go new file mode 100644 index 000000000..9004d92ed --- /dev/null +++ b/test/support/ray_api.go @@ -0,0 +1,45 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package support + +import ( + "github.com/onsi/gomega" +) + +func GetRayJobAPIDetails(t Test, rayClient RayClusterClient, jobID string) *RayJobDetailsResponse { + t.T().Helper() + return RayJobAPIDetails(t, rayClient, jobID)(t) +} + +func WriteRayJobAPILogs(t Test, rayClient RayClusterClient, jobID string) { + t.T().Helper() + logs, err := rayClient.GetJobLogs(jobID) + t.Expect(err).NotTo(gomega.HaveOccurred()) + WriteToOutputDir(t, jobID, Log, []byte(logs)) +} + +func RayJobAPIDetails(t Test, rayClient RayClusterClient, jobID string) func(g gomega.Gomega) *RayJobDetailsResponse { + return func(g gomega.Gomega) *RayJobDetailsResponse { + jobDetails, err := rayClient.GetJobDetails(jobID) + t.Expect(err).NotTo(gomega.HaveOccurred()) + return jobDetails + } +} + +func GetRayJobAPIDetailsStatus(jobDetails *RayJobDetailsResponse) string { + return jobDetails.Status +} diff --git a/test/support/ray_cluster_client.go b/test/support/ray_cluster_client.go new file mode 100644 index 000000000..18dc0c290 --- /dev/null +++ b/test/support/ray_cluster_client.go @@ -0,0 +1,115 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package support + +import ( + "bytes" + "encoding/json" + "io/ioutil" + "net/http" + "net/url" +) + +type RayJobSetup struct { + EntryPoint string `json:"entrypoint"` + RuntimeEnv map[string]any `json:"runtime_env"` +} + +type RayJobResponse struct { + JobID string `json:"job_id"` + SubmissionID string `json:"submission_id"` +} + +type RayJobDetailsResponse struct { + JobID string `json:"job_id"` + SubmissionID string `json:"submission_id"` + Status string `json:"status"` +} + +type RayJobLogsResponse struct { + Logs string `json:"logs"` +} + +var _ RayClusterClient = (*rayClusterClient)(nil) + +type rayClusterClient struct { + endpoint url.URL +} + +type RayClusterClient interface { + CreateJob(job *RayJobSetup) (*RayJobResponse, error) + GetJobDetails(jobID string) (*RayJobDetailsResponse, error) + GetJobLogs(jobID string) (string, error) +} + +func NewRayClusterClient(dashboardEndpoint url.URL) RayClusterClient { + return &rayClusterClient{endpoint: dashboardEndpoint} +} + +func (client *rayClusterClient) CreateJob(job *RayJobSetup) (response *RayJobResponse, err error) { + marshalled, err := json.Marshal(job) + if err != nil { + return + } + + createJobURL := client.endpoint.String() + "/api/jobs/" + resp, err := http.Post(createJobURL, "application/json", bytes.NewReader(marshalled)) + if err != nil { + return + } + + respData, err := ioutil.ReadAll(resp.Body) + if err != nil { + return + } + + err = json.Unmarshal(respData, &response) + return +} + +func (client *rayClusterClient) GetJobDetails(jobID string) (response *RayJobDetailsResponse, err error) { + createJobURL := client.endpoint.String() + "/api/jobs/" + jobID + resp, err := http.Get(createJobURL) + if err != nil { + return + } + + respData, err := ioutil.ReadAll(resp.Body) + if err != nil { + return + } + + err = json.Unmarshal(respData, &response) + return +} + +func (client *rayClusterClient) GetJobLogs(jobID string) (logs string, err error) { + createJobURL := client.endpoint.String() + "/api/jobs/" + jobID + "/logs" + resp, err := http.Get(createJobURL) + if err != nil { + return + } + + respData, err := ioutil.ReadAll(resp.Body) + if err != nil { + return + } + + jobLogs := RayJobLogsResponse{} + err = json.Unmarshal(respData, &jobLogs) + return jobLogs.Logs, err +} diff --git a/test/support/route.go b/test/support/route.go new file mode 100644 index 000000000..b105643fc --- /dev/null +++ b/test/support/route.go @@ -0,0 +1,33 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package support + +import ( + "github.com/onsi/gomega" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + routev1 "github.com/openshift/api/route/v1" +) + +func Route(t Test, namespace, name string) func(g gomega.Gomega) *routev1.Route { + return func(g gomega.Gomega) *routev1.Route { + route, err := t.Client().Route().RouteV1().Routes(namespace).Get(t.Ctx(), name, metav1.GetOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + return route + } +} From 3f5e55d18428313466cfb9567712f0c94fe8c599 Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Thu, 27 Jul 2023 13:55:04 -0400 Subject: [PATCH 083/100] Add Make Target for organizing go imports Signed-off-by: Anish Asthana --- Makefile | 14 +++++++ go.mod | 15 ++++++++ go.sum | 85 ++++++++++++++++++++++++++++++++++++++++++ hack/tools.go | 9 +++++ hack/verify-imports.sh | 13 +++++++ 5 files changed, 136 insertions(+) create mode 100644 hack/tools.go create mode 100755 hack/verify-imports.sh diff --git a/Makefile b/Makefile index 4aea94539..bb9b8ac90 100644 --- a/Makefile +++ b/Makefile @@ -278,6 +278,7 @@ LISTER_GEN ?= $(LOCALBIN)/lister-gen INFORMER_GEN ?= $(LOCALBIN)/informer-gen CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen ENVTEST ?= $(LOCALBIN)/setup-envtest +OPENSHIFT-GOIMPORTS ?= $(LOCALBIN)/openshift-goimports OPERATOR_SDK ?= $(LOCALBIN)/operator-sdk GH_CLI ?= $(LOCALBIN)/gh @@ -338,6 +339,11 @@ envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. $(ENVTEST): $(LOCALBIN) test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest +.PHONY: openshift-goimports +openshift-goimports: $(OPENSHIFT-GOIMPORTS) ## Download openshift-goimports locally if necessary. +$(OPENSHIFT-GOIMPORTS): $(LOCALBIN) + test -s $(LOCALBIN)/openshift-goimports || GOBIN=$(LOCALBIN) go install github.com/openshift-eng/openshift-goimports@latest + OPERATOR_SDK_DL_URL := https://github.com/operator-framework/operator-sdk/releases/download/$(OPERATOR_SDK_VERSION) .PHONY: install-operator-sdk install-operator-sdk: $(OPERATOR_SDK) ## Download fixed version operator-sdk binary for consist outcome @@ -440,3 +446,11 @@ test-e2e: defaults manifests generate fmt vet ## Run e2e tests. .PHONY: setup-e2e setup-e2e: ## Set up e2e tests. KUBERAY_VERSION=$(KUBERAY_VERSION) test/e2e/setup.sh + +.PHONY: imports +imports: openshift-goimports ## Organize imports in go files using openshift-goimports. Example: make imports + $(OPENSHIFT-GOIMPORTS) + +.PHONY: verify-imports +verify-imports: openshift-goimports ## Run import verifications. + ./hack/verify-imports.sh $(OPENSHIFT-GOIMPORTS) diff --git a/go.mod b/go.mod index 8cbc5ec9b..764c68b38 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/manifestival/manifestival v0.7.2 github.com/onsi/ginkgo/v2 v2.9.2 github.com/onsi/gomega v1.27.6 + github.com/openshift-eng/openshift-goimports v0.0.0-20230304234052-c70783e636f2 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c github.com/project-codeflare/multi-cluster-app-dispatcher v1.32.0 @@ -41,22 +42,35 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect github.com/google/uuid v1.1.2 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.12 // indirect + github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/magiconair/properties v1.8.4 // indirect github.com/mailru/easyjson v0.7.6 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pelletier/go-toml v1.8.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.14.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect + github.com/spf13/afero v1.4.1 // indirect + github.com/spf13/cast v1.3.1 // indirect + github.com/spf13/cobra v1.6.0 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/viper v1.7.1 // indirect + github.com/subosito/gotenv v1.2.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect + golang.org/x/mod v0.9.0 // indirect golang.org/x/net v0.8.0 // indirect golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect golang.org/x/sys v0.6.0 // indirect @@ -68,6 +82,7 @@ require ( google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.26.1 // indirect diff --git a/go.sum b/go.sum index 815e362bb..f398a9795 100644 --- a/go.sum +++ b/go.sum @@ -22,6 +22,7 @@ cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4g cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -68,7 +69,10 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= @@ -78,6 +82,7 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -92,6 +97,7 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -100,6 +106,7 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -307,6 +314,8 @@ github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3i github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -317,10 +326,28 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -329,6 +356,8 @@ github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -344,6 +373,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -352,6 +383,7 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -361,6 +393,9 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.4 h1:8KGKTcQQGm0Kv7vEbKFErAoAOFyyacLStRtQSeYtvkY= +github.com/magiconair/properties v1.8.4/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -375,14 +410,25 @@ github.com/manifestival/manifestival v0.6.0/go.mod h1:3Qq9cnPy7sv7FVhg2Kvw0ebb35 github.com/manifestival/manifestival v0.7.2 h1:l4uFdWX/xQK4QcRfqGoMtBvaZeWPEuwD6hVsCwUqZY4= github.com/manifestival/manifestival v0.7.2/go.mod h1:nl3T6HlfHCeidooWVTMI9vYNTBkQ1GdhLNb+smozbdk= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -419,19 +465,26 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= +github.com/openshift-eng/openshift-goimports v0.0.0-20230304234052-c70783e636f2 h1:Zq1BYSO2UmZuu/O1tpYIaC/7ir7ljFqdEY90TwqlseI= +github.com/openshift-eng/openshift-goimports v0.0.0-20230304234052-c70783e636f2/go.mod h1:VV07Ee+14Mjpwh/ZxtwgAieiCvZ20YVUB9jLHixzswk= github.com/openshift/api v0.0.0-20230213134911-7ba313770556 h1:7W2fOhJicyEff24VaF7ASNzPtYvr+iSCVft4SIBAzaE= github.com/openshift/api v0.0.0-20230213134911-7ba313770556/go.mod h1:aQ6LDasvHMvHZXqLHnX2GRmnfTWCF/iIwz8EMTTIE9A= github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c h1:CV76yFOTXmq9VciBR3Bve5ZWzSxdft7gaMVB3kS0rwg= github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c/go.mod h1:lFMO8mLHXWFzSdYvGNo8ivF9SfF6zInA8ZGw4phRnUE= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= +github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/project-codeflare/multi-cluster-app-dispatcher v1.32.0 h1:17lwqvxcWBTi9245lVZgJPvksGwAGA58s8dl7PzpbyE= github.com/project-codeflare/multi-cluster-app-dispatcher v1.32.0/go.mod h1:fmbU5LuV1Z2Sbu1FCEoVuw8qxDFcalXvkPyMfGZHHTc= @@ -471,25 +524,42 @@ github.com/ray-project/kuberay/ray-operator v0.0.0-20230614221720-085c29d40fa9/g github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.4.1 h1:asw9sl74539yqavKaglDM5hFpdJVK0Y5Dr/JOgQ89nQ= +github.com/spf13/afero v1.4.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.6.0 h1:42a0n6jwCot1pUmomAp4T7DeMD+20LFv4Q54pxLf2LI= +github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= +github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -501,6 +571,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -545,12 +617,14 @@ go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -586,12 +660,16 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -654,9 +732,11 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -746,6 +826,7 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -761,6 +842,7 @@ golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -891,6 +973,9 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= +gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= diff --git a/hack/tools.go b/hack/tools.go new file mode 100644 index 000000000..f23ba3623 --- /dev/null +++ b/hack/tools.go @@ -0,0 +1,9 @@ +//go:build tools +// +build tools + +package hack + +// Add tools that hack scripts depend on here, to ensure they are vendored. +import ( + goimports "github.com/openshift-eng/openshift-goimports" +) diff --git a/hack/verify-imports.sh b/hack/verify-imports.sh new file mode 100755 index 000000000..b823a84bb --- /dev/null +++ b/hack/verify-imports.sh @@ -0,0 +1,13 @@ +#!/bin/bash +OPENSHIFT_GOIMPORTS=${1} + +# ${OPENSHIFT_GOIMPORTS} -l +bad_files=$(${OPENSHIFT_GOIMPORTS} -l) + +echo $bad_files +if [[ -n ${bad_files} ]]; then + echo "!!! openshift-goimports needs to be run on the following files:" + echo "${bad_files}" + echo "Try running 'make imports'" + exit 1 +fi From 9b5edf8cbb04c5857aef5608424f72328290cff6 Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Thu, 27 Jul 2023 17:53:32 -0400 Subject: [PATCH 084/100] Organize go imports Signed-off-by: Anish Asthana --- client/applyconfiguration/utils.go | 3 ++- client/clientset/versioned/clientset.go | 3 ++- client/clientset/versioned/fake/clientset_generated.go | 7 ++++--- client/clientset/versioned/fake/register.go | 3 ++- client/clientset/versioned/scheme/register.go | 3 ++- .../typed/codeflare/v1alpha1/codeflare_client.go | 3 ++- .../codeflare/v1alpha1/fake/fake_codeflare_client.go | 3 ++- .../typed/codeflare/v1alpha1/fake/fake_instascale.go | 5 +++-- .../typed/codeflare/v1alpha1/fake/fake_mcad.go | 5 +++-- .../versioned/typed/codeflare/v1alpha1/instascale.go | 7 ++++--- .../versioned/typed/codeflare/v1alpha1/mcad.go | 7 ++++--- .../externalversions/codeflare/v1alpha1/instascale.go | 9 +++++---- .../externalversions/codeflare/v1alpha1/mcad.go | 9 +++++---- client/informer/externalversions/factory.go | 7 ++++--- client/informer/externalversions/generic.go | 3 ++- .../internalinterfaces/factory_interfaces.go | 3 ++- client/listers/codeflare/v1alpha1/instascale.go | 3 ++- client/listers/codeflare/v1alpha1/mcad.go | 3 ++- controllers/config/manifest.go | 1 + controllers/instascale_controller.go | 7 +++---- controllers/instascale_controller_test.go | 5 ++--- controllers/mcad_controller.go | 1 - controllers/mcad_controller_test.go | 5 ++--- controllers/mcad_params.go | 1 + controllers/suite_test.go | 4 +--- main.go | 8 +++----- test/e2e/mnist_pytorch_mcad_job_test.go | 2 +- test/e2e/mnist_raycluster_sdk_test.go | 5 ++--- test/e2e/mnist_rayjob_mcad_raycluster_test.go | 4 ++-- test/support/client.go | 10 +++++----- 30 files changed, 75 insertions(+), 64 deletions(-) diff --git a/client/applyconfiguration/utils.go b/client/applyconfiguration/utils.go index fd87ac36b..44a5e7aa1 100644 --- a/client/applyconfiguration/utils.go +++ b/client/applyconfiguration/utils.go @@ -19,9 +19,10 @@ limitations under the License. package applyconfiguration import ( + schema "k8s.io/apimachinery/pkg/runtime/schema" + v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/applyconfiguration/codeflare/v1alpha1" - schema "k8s.io/apimachinery/pkg/runtime/schema" ) // ForKind returns an apply configuration type for the given GroupVersionKind, or nil if no diff --git a/client/clientset/versioned/clientset.go b/client/clientset/versioned/clientset.go index 1d91a6641..ca6d68fb0 100644 --- a/client/clientset/versioned/clientset.go +++ b/client/clientset/versioned/clientset.go @@ -22,10 +22,11 @@ import ( "fmt" "net/http" - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/typed/codeflare/v1alpha1" discovery "k8s.io/client-go/discovery" rest "k8s.io/client-go/rest" flowcontrol "k8s.io/client-go/util/flowcontrol" + + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/typed/codeflare/v1alpha1" ) type Interface interface { diff --git a/client/clientset/versioned/fake/clientset_generated.go b/client/clientset/versioned/fake/clientset_generated.go index fbbb80dc3..b3e9cbb9a 100644 --- a/client/clientset/versioned/fake/clientset_generated.go +++ b/client/clientset/versioned/fake/clientset_generated.go @@ -19,14 +19,15 @@ limitations under the License. package fake import ( - clientset "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/typed/codeflare/v1alpha1" - fakecodeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/typed/codeflare/v1alpha1/fake" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/discovery" fakediscovery "k8s.io/client-go/discovery/fake" "k8s.io/client-go/testing" + + clientset "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/typed/codeflare/v1alpha1" + fakecodeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/typed/codeflare/v1alpha1/fake" ) // NewSimpleClientset returns a clientset that will respond with the provided objects. diff --git a/client/clientset/versioned/fake/register.go b/client/clientset/versioned/fake/register.go index 4ff1cb3a3..05244eb57 100644 --- a/client/clientset/versioned/fake/register.go +++ b/client/clientset/versioned/fake/register.go @@ -19,12 +19,13 @@ limitations under the License. package fake import ( - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" serializer "k8s.io/apimachinery/pkg/runtime/serializer" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" ) var scheme = runtime.NewScheme() diff --git a/client/clientset/versioned/scheme/register.go b/client/clientset/versioned/scheme/register.go index 743173cea..1a52d7dac 100644 --- a/client/clientset/versioned/scheme/register.go +++ b/client/clientset/versioned/scheme/register.go @@ -19,12 +19,13 @@ limitations under the License. package scheme import ( - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" serializer "k8s.io/apimachinery/pkg/runtime/serializer" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" ) var Scheme = runtime.NewScheme() diff --git a/client/clientset/versioned/typed/codeflare/v1alpha1/codeflare_client.go b/client/clientset/versioned/typed/codeflare/v1alpha1/codeflare_client.go index 4b93fe6fe..bbe389401 100644 --- a/client/clientset/versioned/typed/codeflare/v1alpha1/codeflare_client.go +++ b/client/clientset/versioned/typed/codeflare/v1alpha1/codeflare_client.go @@ -21,9 +21,10 @@ package v1alpha1 import ( "net/http" + rest "k8s.io/client-go/rest" + v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/scheme" - rest "k8s.io/client-go/rest" ) type CodeflareV1alpha1Interface interface { diff --git a/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_codeflare_client.go b/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_codeflare_client.go index 9e234aca7..0a18ef5df 100644 --- a/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_codeflare_client.go +++ b/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_codeflare_client.go @@ -19,9 +19,10 @@ limitations under the License. package fake import ( - v1alpha1 "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/typed/codeflare/v1alpha1" rest "k8s.io/client-go/rest" testing "k8s.io/client-go/testing" + + v1alpha1 "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/typed/codeflare/v1alpha1" ) type FakeCodeflareV1alpha1 struct { diff --git a/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_instascale.go b/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_instascale.go index 8abbdc792..fd6bced0f 100644 --- a/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_instascale.go +++ b/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_instascale.go @@ -23,13 +23,14 @@ import ( json "encoding/json" "fmt" - v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/applyconfiguration/codeflare/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" labels "k8s.io/apimachinery/pkg/labels" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" testing "k8s.io/client-go/testing" + + v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/applyconfiguration/codeflare/v1alpha1" ) // FakeInstaScales implements InstaScaleInterface diff --git a/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_mcad.go b/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_mcad.go index 027f2817d..a06c229ab 100644 --- a/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_mcad.go +++ b/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_mcad.go @@ -23,13 +23,14 @@ import ( json "encoding/json" "fmt" - v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/applyconfiguration/codeflare/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" labels "k8s.io/apimachinery/pkg/labels" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" testing "k8s.io/client-go/testing" + + v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/applyconfiguration/codeflare/v1alpha1" ) // FakeMCADs implements MCADInterface diff --git a/client/clientset/versioned/typed/codeflare/v1alpha1/instascale.go b/client/clientset/versioned/typed/codeflare/v1alpha1/instascale.go index c146a98d2..73cc95d30 100644 --- a/client/clientset/versioned/typed/codeflare/v1alpha1/instascale.go +++ b/client/clientset/versioned/typed/codeflare/v1alpha1/instascale.go @@ -24,13 +24,14 @@ import ( "fmt" "time" - v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/applyconfiguration/codeflare/v1alpha1" - scheme "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/scheme" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" rest "k8s.io/client-go/rest" + + v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/applyconfiguration/codeflare/v1alpha1" + scheme "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/scheme" ) // InstaScalesGetter has a method to return a InstaScaleInterface. diff --git a/client/clientset/versioned/typed/codeflare/v1alpha1/mcad.go b/client/clientset/versioned/typed/codeflare/v1alpha1/mcad.go index 7d70e4035..23587f123 100644 --- a/client/clientset/versioned/typed/codeflare/v1alpha1/mcad.go +++ b/client/clientset/versioned/typed/codeflare/v1alpha1/mcad.go @@ -24,13 +24,14 @@ import ( "fmt" "time" - v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/applyconfiguration/codeflare/v1alpha1" - scheme "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/scheme" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" rest "k8s.io/client-go/rest" + + v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/applyconfiguration/codeflare/v1alpha1" + scheme "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/scheme" ) // MCADsGetter has a method to return a MCADInterface. diff --git a/client/informer/externalversions/codeflare/v1alpha1/instascale.go b/client/informer/externalversions/codeflare/v1alpha1/instascale.go index 70346e653..79d6c7ac9 100644 --- a/client/informer/externalversions/codeflare/v1alpha1/instascale.go +++ b/client/informer/externalversions/codeflare/v1alpha1/instascale.go @@ -22,14 +22,15 @@ import ( "context" time "time" - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" - versioned "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" - internalinterfaces "github.com/project-codeflare/codeflare-operator/client/informer/externalversions/internalinterfaces" - v1alpha1 "github.com/project-codeflare/codeflare-operator/client/listers/codeflare/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" watch "k8s.io/apimachinery/pkg/watch" cache "k8s.io/client-go/tools/cache" + + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" + versioned "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" + internalinterfaces "github.com/project-codeflare/codeflare-operator/client/informer/externalversions/internalinterfaces" + v1alpha1 "github.com/project-codeflare/codeflare-operator/client/listers/codeflare/v1alpha1" ) // InstaScaleInformer provides access to a shared informer and lister for diff --git a/client/informer/externalversions/codeflare/v1alpha1/mcad.go b/client/informer/externalversions/codeflare/v1alpha1/mcad.go index 2fe4d5a30..4cda692b8 100644 --- a/client/informer/externalversions/codeflare/v1alpha1/mcad.go +++ b/client/informer/externalversions/codeflare/v1alpha1/mcad.go @@ -22,14 +22,15 @@ import ( "context" time "time" - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" - versioned "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" - internalinterfaces "github.com/project-codeflare/codeflare-operator/client/informer/externalversions/internalinterfaces" - v1alpha1 "github.com/project-codeflare/codeflare-operator/client/listers/codeflare/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" watch "k8s.io/apimachinery/pkg/watch" cache "k8s.io/client-go/tools/cache" + + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" + versioned "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" + internalinterfaces "github.com/project-codeflare/codeflare-operator/client/informer/externalversions/internalinterfaces" + v1alpha1 "github.com/project-codeflare/codeflare-operator/client/listers/codeflare/v1alpha1" ) // MCADInformer provides access to a shared informer and lister for diff --git a/client/informer/externalversions/factory.go b/client/informer/externalversions/factory.go index e1f7f14a1..0d3733cbe 100644 --- a/client/informer/externalversions/factory.go +++ b/client/informer/externalversions/factory.go @@ -23,13 +23,14 @@ import ( sync "sync" time "time" - versioned "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" - codeflare "github.com/project-codeflare/codeflare-operator/client/informer/externalversions/codeflare" - internalinterfaces "github.com/project-codeflare/codeflare-operator/client/informer/externalversions/internalinterfaces" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" cache "k8s.io/client-go/tools/cache" + + versioned "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" + codeflare "github.com/project-codeflare/codeflare-operator/client/informer/externalversions/codeflare" + internalinterfaces "github.com/project-codeflare/codeflare-operator/client/informer/externalversions/internalinterfaces" ) // SharedInformerOption defines the functional option type for SharedInformerFactory. diff --git a/client/informer/externalversions/generic.go b/client/informer/externalversions/generic.go index 9eea54ebf..6fa936ba3 100644 --- a/client/informer/externalversions/generic.go +++ b/client/informer/externalversions/generic.go @@ -21,9 +21,10 @@ package externalversions import ( "fmt" - v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" schema "k8s.io/apimachinery/pkg/runtime/schema" cache "k8s.io/client-go/tools/cache" + + v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" ) // GenericInformer is type of SharedIndexInformer which will locate and delegate to other diff --git a/client/informer/externalversions/internalinterfaces/factory_interfaces.go b/client/informer/externalversions/internalinterfaces/factory_interfaces.go index b380f668c..0e89555b7 100644 --- a/client/informer/externalversions/internalinterfaces/factory_interfaces.go +++ b/client/informer/externalversions/internalinterfaces/factory_interfaces.go @@ -21,10 +21,11 @@ package internalinterfaces import ( time "time" - versioned "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" cache "k8s.io/client-go/tools/cache" + + versioned "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" ) // NewInformerFunc takes versioned.Interface and time.Duration to return a SharedIndexInformer. diff --git a/client/listers/codeflare/v1alpha1/instascale.go b/client/listers/codeflare/v1alpha1/instascale.go index ee5c22847..c2519af5e 100644 --- a/client/listers/codeflare/v1alpha1/instascale.go +++ b/client/listers/codeflare/v1alpha1/instascale.go @@ -19,10 +19,11 @@ limitations under the License. package v1alpha1 import ( - v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/tools/cache" + + v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" ) // InstaScaleLister helps list InstaScales. diff --git a/client/listers/codeflare/v1alpha1/mcad.go b/client/listers/codeflare/v1alpha1/mcad.go index 8ac44bfce..e2f47832b 100644 --- a/client/listers/codeflare/v1alpha1/mcad.go +++ b/client/listers/codeflare/v1alpha1/mcad.go @@ -19,10 +19,11 @@ limitations under the License. package v1alpha1 import ( - v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/tools/cache" + + v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" ) // MCADLister helps list MCADs. diff --git a/controllers/config/manifest.go b/controllers/config/manifest.go index 2360100db..bfff07c90 100644 --- a/controllers/config/manifest.go +++ b/controllers/config/manifest.go @@ -4,6 +4,7 @@ import ( "github.com/go-logr/logr" mfc "github.com/manifestival/controller-runtime-client" mf "github.com/manifestival/manifestival" + "sigs.k8s.io/controller-runtime/pkg/client" ) diff --git a/controllers/instascale_controller.go b/controllers/instascale_controller.go index 80e38a53d..5fba37e94 100644 --- a/controllers/instascale_controller.go +++ b/controllers/instascale_controller.go @@ -21,6 +21,9 @@ import ( "fmt" "path" + "github.com/go-logr/logr" + mf "github.com/manifestival/manifestival" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" authv1 "k8s.io/api/rbac/v1" @@ -28,7 +31,6 @@ import ( apierrs "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -36,9 +38,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - "github.com/go-logr/logr" - mf "github.com/manifestival/manifestival" - "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" "github.com/project-codeflare/codeflare-operator/controllers/config" "github.com/project-codeflare/codeflare-operator/controllers/util" diff --git a/controllers/instascale_controller_test.go b/controllers/instascale_controller_test.go index 4071c515a..1e880190f 100644 --- a/controllers/instascale_controller_test.go +++ b/controllers/instascale_controller_test.go @@ -3,11 +3,10 @@ package controllers import ( "context" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - mfc "github.com/manifestival/controller-runtime-client" mf "github.com/manifestival/manifestival" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" ) diff --git a/controllers/mcad_controller.go b/controllers/mcad_controller.go index 01f83060d..906bb7bcc 100644 --- a/controllers/mcad_controller.go +++ b/controllers/mcad_controller.go @@ -29,7 +29,6 @@ import ( apierrs "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" diff --git a/controllers/mcad_controller_test.go b/controllers/mcad_controller_test.go index 35e892c35..7ffd30cce 100644 --- a/controllers/mcad_controller_test.go +++ b/controllers/mcad_controller_test.go @@ -3,11 +3,10 @@ package controllers import ( "context" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - mfc "github.com/manifestival/controller-runtime-client" mf "github.com/manifestival/manifestival" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" ) diff --git a/controllers/mcad_params.go b/controllers/mcad_params.go index 5992bede4..846e4c733 100644 --- a/controllers/mcad_params.go +++ b/controllers/mcad_params.go @@ -18,6 +18,7 @@ package controllers import ( mf "github.com/manifestival/manifestival" + mcadv1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" ) diff --git a/controllers/suite_test.go b/controllers/suite_test.go index 074422785..5ee347ba3 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -22,10 +22,9 @@ import ( "testing" "time" + mf "github.com/manifestival/manifestival" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - - mf "github.com/manifestival/manifestival" "go.uber.org/zap/zapcore" appsv1 "k8s.io/api/apps/v1" @@ -36,7 +35,6 @@ import ( "k8s.io/client-go/kubernetes/scheme" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" - ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" diff --git a/main.go b/main.go index 5358e04bd..51d4dbc9b 100644 --- a/main.go +++ b/main.go @@ -23,20 +23,18 @@ import ( "go.uber.org/zap/zapcore" - // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) - // to ensure that exec-entrypoint and run can make use of them. - _ "k8s.io/client-go/plugin/pkg/client/auth" - "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" - + _ "k8s.io/client-go/plugin/pkg/client/auth" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" "github.com/project-codeflare/codeflare-operator/controllers" + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + // to ensure that exec-entrypoint and run can make use of them. // +kubebuilder:scaffold:imports ) diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index 168c7b134..88eca341d 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -20,6 +20,7 @@ import ( "testing" . "github.com/onsi/gomega" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -27,7 +28,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" . "github.com/project-codeflare/codeflare-operator/test/support" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" ) // Trains the MNIST dataset as a batch Job managed by MCAD, and asserts successful completion of the training job. diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index e8bb1f5c7..56385b8fd 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -20,16 +20,15 @@ import ( "testing" . "github.com/onsi/gomega" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" - . "github.com/project-codeflare/codeflare-operator/test/support" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" ) // Creates a Ray cluster, and trains the MNIST dataset using the CodeFlare SDK. diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index 958b30479..72645d5b2 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -21,14 +21,14 @@ import ( "testing" . "github.com/onsi/gomega" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" . "github.com/project-codeflare/codeflare-operator/test/support" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" ) // Trains the MNIST dataset as a RayJob, executed by a Ray cluster managed by MCAD, diff --git a/test/support/client.go b/test/support/client.go index 73931d645..c3c4433ec 100644 --- a/test/support/client.go +++ b/test/support/client.go @@ -17,18 +17,18 @@ limitations under the License. package support import ( - // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) - // to ensure that exec-entrypoint and run can make use of them. - _ "k8s.io/client-go/plugin/pkg/client/auth" + mcadclient "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/client/clientset/controller-versioned" + rayclient "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned" "k8s.io/client-go/kubernetes" + _ "k8s.io/client-go/plugin/pkg/client/auth" "k8s.io/client-go/tools/clientcmd" routev1 "github.com/openshift/client-go/route/clientset/versioned" codeflareclient "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" - mcadclient "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/client/clientset/controller-versioned" - rayclient "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned" + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + // to ensure that exec-entrypoint and run can make use of them. ) type Client interface { From 234cb41a4db0393e4230faaa8628c7615c47e855 Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Thu, 27 Jul 2023 18:47:34 -0400 Subject: [PATCH 085/100] Add verify-imports github action Signed-off-by: Anish Asthana --- .github/workflows/verify_imports.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/workflows/verify_imports.yml diff --git a/.github/workflows/verify_imports.yml b/.github/workflows/verify_imports.yml new file mode 100644 index 000000000..561b3fd88 --- /dev/null +++ b/.github/workflows/verify_imports.yml @@ -0,0 +1,20 @@ +name: Verify organization of imports +on: + push: + branches: + - '**' + paths: + - '**.go' + tags-ignore: + - 'v*' + pull_request: + paths: + - '**.go' +jobs: + verify-imports: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Verify that imports are organized + run: make verify-imports From 223dffe042b39f3ea4278facb15db6d5131ad5a9 Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Fri, 28 Jul 2023 07:32:42 -0400 Subject: [PATCH 086/100] Remove revive check Signed-off-by: Anish Asthana --- .golangci.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.golangci.yaml b/.golangci.yaml index e4fc63b6b..573c82913 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -9,4 +9,3 @@ linters: - staticcheck - typecheck - unused - - revive From ef43fee9249abfbd837c3acde9c3bf9924639629 Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Fri, 28 Jul 2023 08:02:34 -0400 Subject: [PATCH 087/100] Organize go imports for generated files Signed-off-by: Anish Asthana --- Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index bb9b8ac90..a4f47970e 100644 --- a/Makefile +++ b/Makefile @@ -173,8 +173,11 @@ manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and Cust generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." -.PHONY: generate-client ## Generate client packages -generate-client: code-generator +.PHONY: generate-client ## Generate client packages and organize the goimports +generate-client: generate-client-files imports + +.PHONY: generate-client-files +generate-client-files: code-generator rm -rf client $(APPLYCONFIGURATION_GEN) \ --input-dirs="github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" \ From 817f97e2be1832641dabb5ba81844b270419818a Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Fri, 28 Jul 2023 16:08:01 +0200 Subject: [PATCH 088/100] Release workflow - open a PR with changes instead of direct push --- .github/workflows/tag-and-build.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 339d5b6fc..a013291fe 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -48,9 +48,11 @@ jobs: # Permission required to create a release permissions: contents: write + pull-requests: write env: IMAGE_ORG_BASE: quay.io/${{ github.event.inputs.quay-organization }} + PR_BRANCH_NAME: adjustments-release-${{ github.event.inputs.version }} steps: - uses: actions/checkout@v3 @@ -132,6 +134,25 @@ jobs: with: commit_message: Update dependency versions for release ${{ github.event.inputs.version }} file_pattern: 'README.md controllers/defaults.go *.yaml Makefile go.mod go.sum' + create_branch: true + branch: ${{ env.PR_BRANCH_NAME }} + + - name: Create a PR with code changes + run: | + GIT_BRANCH=${GITHUB_REF#refs/heads/} + gh pr create --base "$GIT_BRANCH" --fill --head "${{ env.PR_BRANCH_NAME }}" --label "lgtm" --label "approved" + env: + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + + - name: Wait until PR with code changes is merged + run: | + timeout 3600 bash -c 'until [[ $(gh pr view '${{ env.PR_BRANCH_NAME }}' --json state --jq .state) == "MERGED" ]]; do sleep 5 && echo "$(gh pr view '${{ env.PR_BRANCH_NAME }}' --json state --jq .state)"; done' + env: + GITHUB_TOKEN: ${{ github.TOKEN }} + + - name: Delete remote branch + run: | + git push origin --delete ${{ env.PR_BRANCH_NAME }} - name: Creates a release in GitHub run: | From 9eeacaedf2a23f6889b4fd0e55ff2401e5dc41a9 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Mon, 31 Jul 2023 17:39:30 +0200 Subject: [PATCH 089/100] Fix: add missing --output-base option to applyconfiguration-gen --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index a4f47970e..c6115a28e 100644 --- a/Makefile +++ b/Makefile @@ -183,6 +183,7 @@ generate-client-files: code-generator --input-dirs="github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" \ --go-header-file="hack/boilerplate.go.txt" \ --output-package="github.com/project-codeflare/codeflare-operator/client/applyconfiguration" \ + --output-base="." \ --trim-path-prefix "github.com/project-codeflare/codeflare-operator" $(CLIENT_GEN) \ --input="codeflare/v1alpha1" \ From 47bc1ff1eb91e5ac5d2322b57b3ff263f4236b7c Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Mon, 31 Jul 2023 17:40:30 +0200 Subject: [PATCH 090/100] client: Add missing ControllerImage field to apply configuration --- client/applyconfiguration/codeflare/v1alpha1/mcadspec.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/client/applyconfiguration/codeflare/v1alpha1/mcadspec.go b/client/applyconfiguration/codeflare/v1alpha1/mcadspec.go index c7cb170a5..9358586be 100644 --- a/client/applyconfiguration/codeflare/v1alpha1/mcadspec.go +++ b/client/applyconfiguration/codeflare/v1alpha1/mcadspec.go @@ -33,6 +33,7 @@ type MCADSpecApplyConfiguration struct { QuotaRestURL *string `json:"quotaRestURL,omitempty"` PodCreationTimeout *int `json:"podCreationTimeout,omitempty"` ControllerResources *v1.ResourceRequirements `json:"controllerResources,omitempty"` + ControllerImage *string `json:"controllerImage,omitempty"` } // MCADSpecApplyConfiguration constructs an declarative configuration of the MCADSpec type for use with @@ -104,3 +105,11 @@ func (b *MCADSpecApplyConfiguration) WithControllerResources(value v1.ResourceRe b.ControllerResources = &value return b } + +// WithControllerImage sets the ControllerImage field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the ControllerImage field is set to the value of the last call. +func (b *MCADSpecApplyConfiguration) WithControllerImage(value string) *MCADSpecApplyConfiguration { + b.ControllerImage = &value + return b +} From ca3baf682b6174e0a6424c1fae6734071237630a Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Tue, 27 Jun 2023 11:02:15 +0200 Subject: [PATCH 091/100] Create GitHub action to automate CodeFlare project release --- .../workflows/project-codeflare-release.yml | 126 ++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 .github/workflows/project-codeflare-release.yml diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml new file mode 100644 index 000000000..f3e1b286f --- /dev/null +++ b/.github/workflows/project-codeflare-release.yml @@ -0,0 +1,126 @@ +# This workflow will build and release all components of the CodeFlare project + +name: Project CodeFlare Release +on: + workflow_dispatch: + inputs: + operator-version: + description: 'CodeFlare operator version to be released (for example: v0.0.0)' + required: true + replaces: + description: 'The previous operator semantic version that this release replaces (for example: v0.0.0)' + required: true + mcad-version: + description: 'Version of multi-cluster-app-dispatcher to be released (for example: v0.0.0)' + required: true + codeflare-sdk-version: + description: 'Version of CodeFlare-SDK to be released (for example: v0.0.0)' + required: true + instascale-version: + description: 'Version of InstaScale to be released (for example: v0.0.0)' + required: true + is-stable: + description: 'Select if the built images should be tagged as stable' + required: true + type: boolean + codeflare-repository-organization: + description: 'GitHub organization/user containing repositories used for release' + required: true + default: 'project-codeflare' + quay-organization: + description: 'Quay organization used to push the built images to' + required: true + default: 'project-codeflare' + community-operators-prod-organization: + description: 'Owner of target community-operators-prod repository used to open a PR against' + required: true + default: 'redhat-openshift-ecosystem' + +jobs: + release-mcad: + runs-on: ubuntu-latest + + steps: + - name: Release MCAD + run: | + gh workflow run mcad-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/multi-cluster-app-dispatcher --ref ${{ github.ref }} --field tag=${{ github.event.inputs.mcad-version }} --field quay-organization=${{ github.event.inputs.quay-organization }} + env: + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + shell: bash + + - name: Wait for MCAD run to finish + run: | + # wait for a while for Run to be started + sleep 5 + run_id=$(gh run list --workflow mcad-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/multi-cluster-app-dispatcher --limit 1 --json databaseId --jq .[].databaseId) + gh run watch ${run_id} --repo ${{ github.event.inputs.codeflare-repository-organization }}/multi-cluster-app-dispatcher --interval 10 --exit-status + env: + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + shell: bash + + release-instascale: + needs: release-mcad + runs-on: ubuntu-latest + + steps: + - name: Release InstaScale + run: | + gh workflow run instascale-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/instascale --ref ${{ github.ref }} --field is-stable=${{ github.event.inputs.is-stable }} --field tag=${{ github.event.inputs.instascale-version }} --field mcad-version=${{ github.event.inputs.mcad-version }} --field quay-organization=${{ github.event.inputs.quay-organization }} + env: + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + shell: bash + + - name: Wait for InstaScale run to finish + run: | + # wait for a while for Run to be started + sleep 5 + run_id=$(gh run list --workflow instascale-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/instascale --limit 1 --json databaseId --jq .[].databaseId) + gh run watch ${run_id} --repo ${{ github.event.inputs.codeflare-repository-organization }}/instascale --interval 10 --exit-status + env: + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + shell: bash + + release-codeflare-sdk: + runs-on: ubuntu-latest + + steps: + - name: Release CodeFlare SDK + run: | + semver_version="${{ github.event.inputs.codeflare-sdk-version }}" + plain_version="${semver_version:1}" + gh workflow run release.yaml --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-sdk --ref ${{ github.ref }} --field release-version=${plain_version} --field is-latest=${{ github.event.inputs.is-stable }} --field quay-organization=${{ github.event.inputs.quay-organization }} + env: + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + shell: bash + + - name: Wait for CodeFlare SDK run to finish + run: | + # wait for a while for Run to be started + sleep 5 + run_id=$(gh run list --workflow release.yaml --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-sdk --limit 1 --json databaseId --jq .[].databaseId) + gh run watch ${run_id} --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-sdk --interval 10 --exit-status + env: + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + shell: bash + + release-codeflare-operator: + needs: [release-mcad, release-instascale, release-codeflare-sdk] + runs-on: ubuntu-latest + + steps: + - name: Release CodeFlare operator + run: | + gh workflow run tag-and-build.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator --ref ${{ github.ref }} --field is-stable=${{ github.event.inputs.is-stable }} --field version=${{ github.event.inputs.operator-version }} --field replaces=${{ github.event.inputs.replaces }} --field mcad-version=${{ github.event.inputs.mcad-version }} --field codeflare-sdk-version=${{ github.event.inputs.codeflare-sdk-version }} --field instascale-version=${{ github.event.inputs.instascale-version }} --field quay-organization=${{ github.event.inputs.quay-organization }} --field community-operators-prod-fork-organization=${{ github.event.inputs.codeflare-repository-organization }} --field community-operators-prod-organization=${{ github.event.inputs.community-operators-prod-organization }} + env: + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + shell: bash + + - name: Wait for CodeFlare operator run to finish + run: | + # wait for a while for Run to be started + sleep 5 + run_id=$(gh run list --workflow tag-and-build.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator --limit 1 --json databaseId --jq .[].databaseId) + gh run watch ${run_id} --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator --interval 10 --exit-status + env: + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + shell: bash From c2d2f881549247e067897b83564543be68b126d3 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Mon, 24 Jul 2023 11:27:17 +0100 Subject: [PATCH 092/100] Remove kube-rbac-proxy and open metrics endpoint --- config/default/kustomization.yaml | 8 ++-- config/default/manager_auth_proxy_patch.yaml | 39 ------------------- .../metrics_service.yaml} | 6 +-- config/manager/controller_manager_config.yaml | 2 +- config/manager/manager.yaml | 8 +++- .../rbac/auth_proxy_client_clusterrole.yaml | 9 ----- config/rbac/auth_proxy_role.yaml | 17 -------- config/rbac/auth_proxy_role_binding.yaml | 12 ------ config/rbac/kustomization.yaml | 7 ---- config/rbac/role.yaml | 12 ++++++ controllers/mcad_controller.go | 2 + 11 files changed, 28 insertions(+), 94 deletions(-) delete mode 100644 config/default/manager_auth_proxy_patch.yaml rename config/{rbac/auth_proxy_service.yaml => default/metrics_service.yaml} (79%) delete mode 100644 config/rbac/auth_proxy_client_clusterrole.yaml delete mode 100644 config/rbac/auth_proxy_role.yaml delete mode 100644 config/rbac/auth_proxy_role_binding.yaml diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 1bfe8c541..ad203a943 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -25,11 +25,9 @@ bases: # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. # - ../prometheus -patchesStrategicMerge: -# Protect the /metrics endpoint by putting it behind auth. -# If you want your controller-manager to expose the /metrics -# endpoint w/o any authn/z, please comment the following line. -- manager_auth_proxy_patch.yaml +resources: +# Add metrics service +- metrics_service.yaml # Mount the controller config file for loading manager configurations # through a ComponentConfig type diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml deleted file mode 100644 index 31e1a3648..000000000 --- a/config/default/manager_auth_proxy_patch.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# This patch inject a sidecar container which is a HTTP proxy for the -# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: manager - namespace: system -spec: - template: - spec: - containers: - - name: kube-rbac-proxy - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.0 - args: - - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - - "--v=0" - ports: - - containerPort: 8443 - protocol: TCP - name: https - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - - name: manager - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=0.0.0.0:8080" - - "--leader-elect" diff --git a/config/rbac/auth_proxy_service.yaml b/config/default/metrics_service.yaml similarity index 79% rename from config/rbac/auth_proxy_service.yaml rename to config/default/metrics_service.yaml index c53c92d18..0f420610a 100644 --- a/config/rbac/auth_proxy_service.yaml +++ b/config/default/metrics_service.yaml @@ -5,10 +5,10 @@ metadata: namespace: system spec: ports: - - name: https - port: 8443 + - name: metrics + port: 8080 protocol: TCP - targetPort: 8080 + targetPort: metrics selector: app.kubernetes.io/name: codeflare-operator app.kubernetes.io/part-of: codeflare diff --git a/config/manager/controller_manager_config.yaml b/config/manager/controller_manager_config.yaml index ea5cfaa9d..7741092eb 100644 --- a/config/manager/controller_manager_config.yaml +++ b/config/manager/controller_manager_config.yaml @@ -3,7 +3,7 @@ kind: ControllerManagerConfig health: healthProbeBindAddress: :8081 metrics: - bindAddress: 127.0.0.1:8080 + bindAddress: 0.0.0.0:8080 webhook: port: 9443 leaderElection: diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 59b41783f..a8467ccf4 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -35,7 +35,9 @@ spec: - command: - /manager args: - - --leader-elect + - "--health-probe-bind-address=:8081" + - "--metrics-bind-address=0.0.0.0:8080" + - "--leader-elect" image: controller:latest imagePullPolicy: Always name: manager @@ -44,6 +46,10 @@ spec: capabilities: drop: - "ALL" + ports: + - containerPort: 8080 + protocol: TCP + name: metrics livenessProbe: httpGet: path: /healthz diff --git a/config/rbac/auth_proxy_client_clusterrole.yaml b/config/rbac/auth_proxy_client_clusterrole.yaml deleted file mode 100644 index 51a75db47..000000000 --- a/config/rbac/auth_proxy_client_clusterrole.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: metrics-reader -rules: -- nonResourceURLs: - - "/metrics" - verbs: - - get diff --git a/config/rbac/auth_proxy_role.yaml b/config/rbac/auth_proxy_role.yaml deleted file mode 100644 index 80e1857c5..000000000 --- a/config/rbac/auth_proxy_role.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create diff --git a/config/rbac/auth_proxy_role_binding.yaml b/config/rbac/auth_proxy_role_binding.yaml deleted file mode 100644 index ec7acc0a1..000000000 --- a/config/rbac/auth_proxy_role_binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: proxy-role -subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 8a599efd0..3c96e8178 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -10,10 +10,3 @@ resources: - edit_role_binding.yaml # We are using this binding as mcad requires this role - leader_election_role.yaml - leader_election_role_binding.yaml -# Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) -# which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index d0c21ba62..9805d333c 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -44,6 +44,18 @@ rules: - patch - update - watch +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create - apiGroups: - codeflare.codeflare.dev resources: diff --git a/controllers/mcad_controller.go b/controllers/mcad_controller.go index 906bb7bcc..770df03e0 100644 --- a/controllers/mcad_controller.go +++ b/controllers/mcad_controller.go @@ -123,6 +123,8 @@ func (r *MCADReconciler) DeleteResource(params *MCADParams, template string, fns // +kubebuilder:rbac:groups=extensions,resources=replicasets,verbs=get;list;watch // +kubebuilder:rbac:groups=policy,resources=poddisruptionbudgets,verbs=get;list;watch // +kubebuilder:rbac:groups=storage.k8s.io,resources=csidrivers;csinodes;csistoragecapacities,verbs=get;list;watch +// +kubebuilder:rbac:groups=authentication.k8s.io,resources=tokenreviews,verbs=create +// +kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create func (r *MCADReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.Log.WithValues("namespace", req.Namespace) From 8bf0aab426ecc68287697505fecd20e17d72d1b6 Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Mon, 31 Jul 2023 16:02:02 -0400 Subject: [PATCH 093/100] Create consolidated action for verification of auto-generated / formatted artifacts Signed-off-by: Anish Asthana --- .github/workflows/verify_generated_files.yml | 44 ++++++++++++++++++++ .github/workflows/verify_imports.yml | 20 --------- 2 files changed, 44 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/verify_generated_files.yml delete mode 100644 .github/workflows/verify_imports.yml diff --git a/.github/workflows/verify_generated_files.yml b/.github/workflows/verify_generated_files.yml new file mode 100644 index 000000000..5783d4253 --- /dev/null +++ b/.github/workflows/verify_generated_files.yml @@ -0,0 +1,44 @@ +name: Verify Generated Files and Import Organization +on: + push: + branches: + - '**' + paths: + - '**.go' + - '**go.mod' + - '**go.sum' + tags-ignore: + - 'v*' + pull_request: + paths: + - '**.go' + - '**go.mod' + - '**go.sum' +jobs: + verify-generated-functions: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Verify that the DeepCopy, DeepCopyInto, and DeepCopyObject method implementations have been generated + run: make generate && git diff --exit-code + + verify-generate-client: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Verify that the latest client has been generated + run: make generate-client && git diff --exit-code + + verify-imports: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Verify that imports are organized + run: make verify-imports + + verify-manifests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Verify that the latest WebhookConfigurations, ClusterRoles, and CustomResourceDefinitions have been generated + run: make manifests && git diff --exit-code diff --git a/.github/workflows/verify_imports.yml b/.github/workflows/verify_imports.yml deleted file mode 100644 index 561b3fd88..000000000 --- a/.github/workflows/verify_imports.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Verify organization of imports -on: - push: - branches: - - '**' - paths: - - '**.go' - tags-ignore: - - 'v*' - pull_request: - paths: - - '**.go' -jobs: - verify-imports: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Verify that imports are organized - run: make verify-imports From 226a5609f9eef63986d7f8a9b7e5f251e16bce2d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Aug 2023 20:00:29 +0000 Subject: [PATCH 094/100] Bump github.com/onsi/ginkgo/v2 from 2.9.2 to 2.11.0 Bumps [github.com/onsi/ginkgo/v2](https://github.com/onsi/ginkgo) from 2.9.2 to 2.11.0. - [Release notes](https://github.com/onsi/ginkgo/releases) - [Changelog](https://github.com/onsi/ginkgo/blob/master/CHANGELOG.md) - [Commits](https://github.com/onsi/ginkgo/compare/v2.9.2...v2.11.0) --- updated-dependencies: - dependency-name: github.com/onsi/ginkgo/v2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 18 +++++++++--------- go.sum | 36 ++++++++++++++++++------------------ 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/go.mod b/go.mod index 764c68b38..c144579cd 100644 --- a/go.mod +++ b/go.mod @@ -6,13 +6,13 @@ require ( github.com/go-logr/logr v1.2.4 github.com/manifestival/controller-runtime-client v0.4.0 github.com/manifestival/manifestival v0.7.2 - github.com/onsi/ginkgo/v2 v2.9.2 - github.com/onsi/gomega v1.27.6 + github.com/onsi/ginkgo/v2 v2.11.0 + github.com/onsi/gomega v1.27.8 github.com/openshift-eng/openshift-goimports v0.0.0-20230304234052-c70783e636f2 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c github.com/project-codeflare/multi-cluster-app-dispatcher v1.32.0 - github.com/ray-project/kuberay/ray-operator v0.0.0-20230614221720-085c29d40fa9 + github.com/ray-project/kuberay/ray-operator v0.0.0-20230802222355-153f35c9fd14 go.uber.org/zap v1.24.0 k8s.io/api v0.26.3 k8s.io/apimachinery v0.26.3 @@ -70,14 +70,14 @@ require ( github.com/subosito/gotenv v1.2.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect - golang.org/x/mod v0.9.0 // indirect - golang.org/x/net v0.8.0 // indirect + golang.org/x/mod v0.10.0 // indirect + golang.org/x/net v0.12.0 // indirect golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect - golang.org/x/sys v0.6.0 // indirect - golang.org/x/term v0.6.0 // indirect - golang.org/x/text v0.8.0 // indirect + golang.org/x/sys v0.10.0 // indirect + golang.org/x/term v0.10.0 // indirect + golang.org/x/text v0.11.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.7.0 // indirect + golang.org/x/tools v0.9.3 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/go.sum b/go.sum index f398a9795..ad4b81b95 100644 --- a/go.sum +++ b/go.sum @@ -455,16 +455,16 @@ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo/v2 v2.9.2 h1:BA2GMJOtfGAfagzYtrAlufIP0lq6QERkFmHLMLPwFSU= -github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxeMeDuts= +github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= +github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= -github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= +github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= +github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ= github.com/openshift-eng/openshift-goimports v0.0.0-20230304234052-c70783e636f2 h1:Zq1BYSO2UmZuu/O1tpYIaC/7ir7ljFqdEY90TwqlseI= github.com/openshift-eng/openshift-goimports v0.0.0-20230304234052-c70783e636f2/go.mod h1:VV07Ee+14Mjpwh/ZxtwgAieiCvZ20YVUB9jLHixzswk= github.com/openshift/api v0.0.0-20230213134911-7ba313770556 h1:7W2fOhJicyEff24VaF7ASNzPtYvr+iSCVft4SIBAzaE= @@ -519,8 +519,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/ray-project/kuberay/ray-operator v0.0.0-20230614221720-085c29d40fa9 h1:qIThU9GGqEay/y78y4Y9e1FVfrdkH5MFnT0zEJ9yh0A= -github.com/ray-project/kuberay/ray-operator v0.0.0-20230614221720-085c29d40fa9/go.mod h1:2auArgwD9dXXJz1oc7SqQ4U/rHdpwnrBwG98kr8OWXA= +github.com/ray-project/kuberay/ray-operator v0.0.0-20230802222355-153f35c9fd14 h1:KnqFh/uiqY7Zdwr9fUaefMa4j0rYSLkk9tuE4lmW4Ec= +github.com/ray-project/kuberay/ray-operator v0.0.0-20230802222355-153f35c9fd14/go.mod h1:hqphTv0O5l6hHf4/OtEn4ie/OHPoVydLograwaK5cHI= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -660,8 +660,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= -golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -709,8 +709,8 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -788,14 +788,14 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= +golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -805,8 +805,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -871,8 +871,8 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= -golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= +golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 8a9d684b631e69b7c919fdab0b47834ce1f4179f Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 8 Aug 2023 15:02:07 +0200 Subject: [PATCH 095/100] Upgrade MCAD to version 1.33.0 --- Makefile | 3 ++- config/crd/mcad/kustomization.yaml | 2 +- controllers/defaults.go | 2 +- go.mod | 4 ++-- go.sum | 8 ++++---- test/e2e/mnist_pytorch_mcad_job_test.go | 2 +- test/e2e/mnist_rayjob_mcad_raycluster_test.go | 2 +- test/support/client.go | 2 +- test/support/mcad.go | 2 +- 9 files changed, 14 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index c6115a28e..b6be34ce1 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ BUNDLE_VERSION ?= $(VERSION:v%=%) INSTASCALE_VERSION ?= v0.0.5 # MCAD_VERSION defines the default version of the MCAD controller -MCAD_VERSION ?= v1.32.0 +MCAD_VERSION ?= v1.33.0 # MCAD_REF, MCAD_REPO and MCAD_CRD define the reference to MCAD CRD resources MCAD_REF ?= release-${MCAD_VERSION} MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher @@ -223,6 +223,7 @@ vet: ## Run go vet against code. .PHONY: modules modules: ## Update Go dependencies. go get $(MCAD_REPO)@$(MCAD_VERSION) + go get github.com/ray-project/kuberay/ray-operator .PHONY: build build: modules defaults generate fmt vet ## Build manager binary. diff --git a/config/crd/mcad/kustomization.yaml b/config/crd/mcad/kustomization.yaml index a433f8335..8ea96b055 100644 --- a/config/crd/mcad/kustomization.yaml +++ b/config/crd/mcad/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=release-v0.0.0 # kpt-set: ${MCAD_CRD} +- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v0.0.0 # kpt-set: ${MCAD_CRD} diff --git a/controllers/defaults.go b/controllers/defaults.go index 4dc0b777b..3a0a746dd 100644 --- a/controllers/defaults.go +++ b/controllers/defaults.go @@ -5,6 +5,6 @@ package controllers // *********************** const ( - MCADImage = "quay.io/project-codeflare/mcad-controller:release-v1.32.0" + MCADImage = "quay.io/project-codeflare/mcad-controller:release-v1.33.0" InstaScaleImage = "quay.io/project-codeflare/instascale-controller:v0.0.5" ) diff --git a/go.mod b/go.mod index c144579cd..2987d8fd5 100644 --- a/go.mod +++ b/go.mod @@ -11,8 +11,8 @@ require ( github.com/openshift-eng/openshift-goimports v0.0.0-20230304234052-c70783e636f2 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/multi-cluster-app-dispatcher v1.32.0 - github.com/ray-project/kuberay/ray-operator v0.0.0-20230802222355-153f35c9fd14 + github.com/project-codeflare/multi-cluster-app-dispatcher v1.33.0 + github.com/ray-project/kuberay/ray-operator v0.0.0-20230807232553-238cb4e945b6 go.uber.org/zap v1.24.0 k8s.io/api v0.26.3 k8s.io/apimachinery v0.26.3 diff --git a/go.sum b/go.sum index ad4b81b95..4e90f974b 100644 --- a/go.sum +++ b/go.sum @@ -486,8 +486,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.32.0 h1:17lwqvxcWBTi9245lVZgJPvksGwAGA58s8dl7PzpbyE= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.32.0/go.mod h1:fmbU5LuV1Z2Sbu1FCEoVuw8qxDFcalXvkPyMfGZHHTc= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.33.0 h1:6a+MnxcFSlheC7RIPGg3s/QCt5+7dD8mJKwdpST7i70= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.33.0/go.mod h1:0J0BDSaIN5lvlmgw+32FcMqe8SflXHtHByUbHmPl4w8= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -519,8 +519,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/ray-project/kuberay/ray-operator v0.0.0-20230802222355-153f35c9fd14 h1:KnqFh/uiqY7Zdwr9fUaefMa4j0rYSLkk9tuE4lmW4Ec= -github.com/ray-project/kuberay/ray-operator v0.0.0-20230802222355-153f35c9fd14/go.mod h1:hqphTv0O5l6hHf4/OtEn4ie/OHPoVydLograwaK5cHI= +github.com/ray-project/kuberay/ray-operator v0.0.0-20230807232553-238cb4e945b6 h1:0O9dKI7eCPJIzJD/HfzbQgACq4GTr4j+VpwjdS7Qv0w= +github.com/ray-project/kuberay/ray-operator v0.0.0-20230807232553-238cb4e945b6/go.mod h1:hqphTv0O5l6hHf4/OtEn4ie/OHPoVydLograwaK5cHI= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index 88eca341d..b2b7fa13c 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -137,7 +137,7 @@ func TestMNISTPyTorchMCAD(t *testing.T) { }, } - _, err = test.Client().MCAD().ArbV1().AppWrappers(namespace.Name).Create(aw) + _, err = test.Client().MCAD().McadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created MCAD %s/%s successfully", aw.Namespace, aw.Name) diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index 72645d5b2..e6aaea8f1 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -212,7 +212,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { }, } - aw, err = test.Client().MCAD().ArbV1().AppWrappers(namespace.Name).Create(aw) + aw, err = test.Client().MCAD().McadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created MCAD %s/%s successfully", aw.Namespace, aw.Name) diff --git a/test/support/client.go b/test/support/client.go index c3c4433ec..86a895eec 100644 --- a/test/support/client.go +++ b/test/support/client.go @@ -17,7 +17,7 @@ limitations under the License. package support import ( - mcadclient "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/client/clientset/controller-versioned" + mcadclient "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/client/clientset/versioned" rayclient "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned" "k8s.io/client-go/kubernetes" diff --git a/test/support/mcad.go b/test/support/mcad.go index 607a94f74..a4d69b759 100644 --- a/test/support/mcad.go +++ b/test/support/mcad.go @@ -26,7 +26,7 @@ import ( func AppWrapper(t Test, namespace *corev1.Namespace, name string) func(g gomega.Gomega) *mcadv1beta1.AppWrapper { return func(g gomega.Gomega) *mcadv1beta1.AppWrapper { - aw, err := t.Client().MCAD().ArbV1().AppWrappers(namespace.Name).Get(name, metav1.GetOptions{}) + aw, err := t.Client().MCAD().McadV1beta1().AppWrappers(namespace.Name).Get(t.Ctx(), name, metav1.GetOptions{}) g.Expect(err).NotTo(gomega.HaveOccurred()) return aw } From cd8b115be6213bc99c37ffa68f5a947708f6f9f8 Mon Sep 17 00:00:00 2001 From: codeflare-machine-account Date: Tue, 8 Aug 2023 14:57:51 +0000 Subject: [PATCH 096/100] Update dependency versions for release v0.1.0 --- Makefile | 2 +- README.md | 8 ++++---- controllers/defaults.go | 2 +- .../instascale_test_results/case_1/deployment.yaml | 2 +- .../instascale_test_results/case_2/deployment.yaml | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index b6be34ce1..e0c85fb46 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # INSTASCALE_VERSION defines the default version of the InstaScale controller -INSTASCALE_VERSION ?= v0.0.5 +INSTASCALE_VERSION ?= v0.0.6 # MCAD_VERSION defines the default version of the MCAD controller MCAD_VERSION ?= v1.33.0 diff --git a/README.md b/README.md index e5f25443e..e434d001d 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,10 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------| -| CodeFlare Operator | v0.0.6 | -| Multi-Cluster App Dispatcher | v1.32.0 | -| CodeFlare-SDK | v0.5.0 | -| InstaScale | v0.0.5 | +| CodeFlare Operator | v0.1.0 | +| Multi-Cluster App Dispatcher | v1.33.0 | +| CodeFlare-SDK | v0.6.1 | +| InstaScale | v0.0.6 | | KubeRay | v0.5.0 | diff --git a/controllers/defaults.go b/controllers/defaults.go index 3a0a746dd..026be9dff 100644 --- a/controllers/defaults.go +++ b/controllers/defaults.go @@ -6,5 +6,5 @@ package controllers const ( MCADImage = "quay.io/project-codeflare/mcad-controller:release-v1.33.0" - InstaScaleImage = "quay.io/project-codeflare/instascale-controller:v0.0.5" + InstaScaleImage = "quay.io/project-codeflare/instascale-controller:v0.0.6" ) diff --git a/controllers/testdata/instascale_test_results/case_1/deployment.yaml b/controllers/testdata/instascale_test_results/case_1/deployment.yaml index bb398dbbd..134702669 100644 --- a/controllers/testdata/instascale_test_results/case_1/deployment.yaml +++ b/controllers/testdata/instascale_test_results/case_1/deployment.yaml @@ -19,7 +19,7 @@ spec: - name: instascale args: - "--configs-namespace=default" - image: quay.io/project-codeflare/instascale-controller:v0.0.5 + image: quay.io/project-codeflare/instascale-controller:v0.0.6 resources: limits: cpu: '2' diff --git a/controllers/testdata/instascale_test_results/case_2/deployment.yaml b/controllers/testdata/instascale_test_results/case_2/deployment.yaml index ef7b8da11..2916c7195 100644 --- a/controllers/testdata/instascale_test_results/case_2/deployment.yaml +++ b/controllers/testdata/instascale_test_results/case_2/deployment.yaml @@ -19,7 +19,7 @@ spec: - name: instascale args: - "--configs-namespace=default" - image: quay.io/project-codeflare/instascale-controller:v0.0.5 + image: quay.io/project-codeflare/instascale-controller:v0.0.6 resources: limits: cpu: '1' From 81e8f817598f0fb139f15f57e095f6433ce31308 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 3 Aug 2023 10:22:22 +0200 Subject: [PATCH 097/100] e2e: Use Ray dashboard API to retrieve job logs --- .github/actions/kind/action.yml | 9 ++ test/e2e/mnist_rayjob_mcad_raycluster_test.go | 103 +++++++++++++++++- test/support/ingress.go | 41 +++++++ test/support/ray.go | 26 +---- test/support/ray_cluster_client.go | 8 +- test/support/route.go | 5 + 6 files changed, 164 insertions(+), 28 deletions(-) create mode 100644 test/support/ingress.go diff --git a/.github/actions/kind/action.yml b/.github/actions/kind/action.yml index 268c48a51..59dcafef7 100644 --- a/.github/actions/kind/action.yml +++ b/.github/actions/kind/action.yml @@ -47,3 +47,12 @@ runs: echo "KinD cluster:" kubectl cluster-info kubectl describe nodes + + - name: Install Ingress controller + shell: bash + run: | + VERSION=controller-v1.6.4 + echo "Deploying Ingress controller into KinD cluster" + curl https://raw.githubusercontent.com/kubernetes/ingress-nginx/"${VERSION}"/deploy/static/provider/kind/deploy.yaml | sed "s/--publish-status-address=localhost/--report-node-internal-ip-address\\n - --status-update-interval=10/g" | kubectl apply -f - + kubectl annotate ingressclass nginx "ingressclass.kubernetes.io/is-default-class=true" + kubectl -n ingress-nginx wait --timeout=300s --for=condition=Available deployments --all diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index e6aaea8f1..037a12293 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -18,6 +18,7 @@ package e2e import ( "encoding/base64" + "net/url" "testing" . "github.com/onsi/gomega" @@ -25,8 +26,12 @@ import ( rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + + routev1 "github.com/openshift/api/route/v1" . "github.com/project-codeflare/codeflare-operator/test/support" ) @@ -252,8 +257,104 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created RayJob %s/%s successfully", rayJob.Namespace, rayJob.Name) + var rayDashboardURL url.URL + if IsOpenShift(test) { + // Create a route to expose the Ray cluster API + route := &routev1.Route{ + TypeMeta: metav1.TypeMeta{ + APIVersion: routev1.GroupVersion.String(), + Kind: "Route", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace.Name, + Name: "ray-dashboard", + }, + Spec: routev1.RouteSpec{ + To: routev1.RouteTargetReference{ + Name: "raycluster-head-svc", + }, + Port: &routev1.RoutePort{ + TargetPort: intstr.FromString("dashboard"), + }, + }, + } + + _, err := test.Client().Route().RouteV1().Routes(namespace.Name).Create(test.Ctx(), route, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created Route %s/%s successfully", route.Namespace, route.Name) + + test.T().Logf("Waiting for Route %s/%s to be admitted", route.Namespace, route.Name) + test.Eventually(Route(test, route.Namespace, route.Name), TestTimeoutMedium). + Should(WithTransform(ConditionStatus(routev1.RouteAdmitted), Equal(corev1.ConditionTrue))) + + route = GetRoute(test, route.Namespace, route.Name) + + rayDashboardURL = url.URL{ + Scheme: "http", + Host: route.Status.Ingress[0].Host, + } + } else { + ingress := &networkingv1.Ingress{ + TypeMeta: metav1.TypeMeta{ + APIVersion: networkingv1.SchemeGroupVersion.String(), + Kind: "Ingress", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace.Name, + Name: "ray-dashboard", + Annotations: map[string]string{ + "nginx.ingress.kubernetes.io/use-regex": "true", + "nginx.ingress.kubernetes.io/rewrite-target": "/$2", + }, + }, + Spec: networkingv1.IngressSpec{ + Rules: []networkingv1.IngressRule{ + { + IngressRuleValue: networkingv1.IngressRuleValue{ + HTTP: &networkingv1.HTTPIngressRuleValue{ + Paths: []networkingv1.HTTPIngressPath{ + { + Path: "/ray-dashboard(/|$)(.*)", + PathType: Ptr(networkingv1.PathTypePrefix), + Backend: networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: "raycluster-head-svc", + Port: networkingv1.ServiceBackendPort{ + Name: "dashboard", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + _, err := test.Client().Core().NetworkingV1().Ingresses(ingress.Namespace).Create(test.Ctx(), ingress, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created Ingress %s/%s successfully", ingress.Namespace, ingress.Name) + + test.T().Logf("Waiting for Ingress %s/%s to be admitted", ingress.Namespace, ingress.Name) + test.Eventually(Ingress(test, ingress.Namespace, ingress.Name), TestTimeoutMedium). + Should(WithTransform(LoadBalancerIngresses, HaveLen(1))) + + ingress = GetIngress(test, ingress.Namespace, ingress.Name) + + rayDashboardURL = url.URL{ + Scheme: "http", + Host: ingress.Status.LoadBalancer.Ingress[0].IP, + Path: "ray-dashboard", + } + } + + test.T().Logf("Connecting to Ray cluster at: %s", rayDashboardURL.String()) + rayClient := NewRayClusterClient(rayDashboardURL) + // Retrieving the job logs once it has completed or timed out - defer WriteRayJobLogs(test, rayJob.Namespace, rayJob.Name) + defer WriteRayJobAPILogs(test, rayClient, GetRayJobId(test, rayJob.Namespace, rayJob.Name)) test.T().Logf("Waiting for RayJob %s/%s to complete", rayJob.Namespace, rayJob.Name) test.Eventually(RayJob(test, rayJob.Namespace, rayJob.Name), TestTimeoutLong). diff --git a/test/support/ingress.go b/test/support/ingress.go new file mode 100644 index 000000000..667abe6e4 --- /dev/null +++ b/test/support/ingress.go @@ -0,0 +1,41 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package support + +import ( + "github.com/onsi/gomega" + + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func Ingress(t Test, namespace, name string) func(g gomega.Gomega) *networkingv1.Ingress { + return func(g gomega.Gomega) *networkingv1.Ingress { + ingress, err := t.Client().Core().NetworkingV1().Ingresses(namespace).Get(t.Ctx(), name, metav1.GetOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + return ingress + } +} + +func GetIngress(t Test, namespace, name string) *networkingv1.Ingress { + t.T().Helper() + return Ingress(t, namespace, name)(t) +} + +func LoadBalancerIngresses(ingress *networkingv1.Ingress) []networkingv1.IngressLoadBalancerIngress { + return ingress.Status.LoadBalancer.Ingress +} diff --git a/test/support/ray.go b/test/support/ray.go index 4dc6ebc1b..fec4ba672 100644 --- a/test/support/ray.go +++ b/test/support/ray.go @@ -17,8 +17,6 @@ limitations under the License. package support import ( - "encoding/json" - "github.com/onsi/gomega" rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" @@ -44,28 +42,10 @@ func RayJobStatus(job *rayv1alpha1.RayJob) rayv1alpha1.JobStatus { return job.Status.JobStatus } -func GetRayJobLogs(t Test, namespace, name string) []byte { +func GetRayJobId(t Test, namespace, name string) string { t.T().Helper() - - job := GetRayJob(t, namespace, name) - - response := t.Client().Core().CoreV1().RESTClient(). - Get(). - AbsPath("/api/v1/namespaces", job.Namespace, "services", "http:"+job.Status.RayClusterName+"-head-svc:dashboard", "proxy", "api", "jobs", job.Status.JobId, "logs"). - Do(t.Ctx()) - t.Expect(response.Error()).NotTo(gomega.HaveOccurred()) - - body := map[string]string{} - bytes, _ := response.Raw() - t.Expect(json.Unmarshal(bytes, &body)).To(gomega.Succeed()) - t.Expect(body).To(gomega.HaveKey("logs")) - - return []byte(body["logs"]) -} - -func WriteRayJobLogs(t Test, namespace, name string) { - t.T().Logf("Retrieving RayJob %s/%s logs", namespace, name) - WriteToOutputDir(t, name, Log, GetRayJobLogs(t, namespace, name)) + job := RayJob(t, namespace, name)(t) + return job.Status.JobId } func RayCluster(t Test, namespace, name string) func(g gomega.Gomega) *rayv1alpha1.RayCluster { diff --git a/test/support/ray_cluster_client.go b/test/support/ray_cluster_client.go index 18dc0c290..ce36459bd 100644 --- a/test/support/ray_cluster_client.go +++ b/test/support/ray_cluster_client.go @@ -19,7 +19,7 @@ package support import ( "bytes" "encoding/json" - "io/ioutil" + "io" "net/http" "net/url" ) @@ -72,7 +72,7 @@ func (client *rayClusterClient) CreateJob(job *RayJobSetup) (response *RayJobRes return } - respData, err := ioutil.ReadAll(resp.Body) + respData, err := io.ReadAll(resp.Body) if err != nil { return } @@ -88,7 +88,7 @@ func (client *rayClusterClient) GetJobDetails(jobID string) (response *RayJobDet return } - respData, err := ioutil.ReadAll(resp.Body) + respData, err := io.ReadAll(resp.Body) if err != nil { return } @@ -104,7 +104,7 @@ func (client *rayClusterClient) GetJobLogs(jobID string) (logs string, err error return } - respData, err := ioutil.ReadAll(resp.Body) + respData, err := io.ReadAll(resp.Body) if err != nil { return } diff --git a/test/support/route.go b/test/support/route.go index b105643fc..8a74ca12f 100644 --- a/test/support/route.go +++ b/test/support/route.go @@ -31,3 +31,8 @@ func Route(t Test, namespace, name string) func(g gomega.Gomega) *routev1.Route return route } } + +func GetRoute(t Test, namespace, name string) *routev1.Route { + t.T().Helper() + return Route(t, namespace, name)(t) +} From bc4b46a6818a04ff7243bff8903fa08b771d53aa Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 3 Aug 2023 11:16:36 +0200 Subject: [PATCH 098/100] e2e: Add kind-e2e target to setup test KinD cluster --- Makefile | 4 ++++ README.md | 6 +++++- test/e2e/kind.sh | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) create mode 100755 test/e2e/kind.sh diff --git a/Makefile b/Makefile index e0c85fb46..77b32679a 100644 --- a/Makefile +++ b/Makefile @@ -448,6 +448,10 @@ test-unit: defaults manifests generate fmt vet envtest ## Run unit tests. test-e2e: defaults manifests generate fmt vet ## Run e2e tests. go test -timeout 30m -v ./test/e2e +.PHONY: kind-e2e +setup-e2e: ## Set up e2e KinD cluster. + test/e2e/kind.sh + .PHONY: setup-e2e setup-e2e: ## Set up e2e tests. KUBERAY_VERSION=$(KUBERAY_VERSION) test/e2e/setup.sh diff --git a/README.md b/README.md index e434d001d..5cdb6e1c5 100644 --- a/README.md +++ b/README.md @@ -25,11 +25,15 @@ The e2e tests can be executed locally by running the following commands: ```bash # Create a KinD cluster - $ kind create cluster --image kindest/node:v1.25.8 + $ make kind-e2e # Install the CRDs $ make install ``` + [!NOTE] + Some e2e tests cover the access to services via Ingresses, as end-users would do, which requires access to the Ingress controller load balancer by its IP. + For it to work on macOS, this requires installing [docker-mac-net-connect](https://github.com/chipmk/docker-mac-net-connect). + 2. Start the operator locally: ```bash diff --git a/test/e2e/kind.sh b/test/e2e/kind.sh new file mode 100755 index 000000000..5b39a164e --- /dev/null +++ b/test/e2e/kind.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Copyright 2022 IBM, Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail +: "${INGRESS_NGINX_VERSION:=controller-v1.6.4}" + +echo "Creating KinD cluster" +cat < Date: Wed, 9 Aug 2023 10:13:11 +0100 Subject: [PATCH 099/100] Small Make target bug-fix --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 77b32679a..a3ff4efdf 100644 --- a/Makefile +++ b/Makefile @@ -449,7 +449,7 @@ test-e2e: defaults manifests generate fmt vet ## Run e2e tests. go test -timeout 30m -v ./test/e2e .PHONY: kind-e2e -setup-e2e: ## Set up e2e KinD cluster. +kind-e2e: ## Set up e2e KinD cluster. test/e2e/kind.sh .PHONY: setup-e2e From 28726c32e618dd2775219933768174bf3056ea87 Mon Sep 17 00:00:00 2001 From: Dimitri Saridakis Date: Wed, 9 Aug 2023 16:13:48 +0100 Subject: [PATCH 100/100] docs: remove dollar signs to allow for correct copy and paste --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 5cdb6e1c5..f5349d59c 100644 --- a/README.md +++ b/README.md @@ -25,9 +25,9 @@ The e2e tests can be executed locally by running the following commands: ```bash # Create a KinD cluster - $ make kind-e2e + make kind-e2e # Install the CRDs - $ make install + make install ``` [!NOTE] @@ -37,7 +37,7 @@ The e2e tests can be executed locally by running the following commands: 2. Start the operator locally: ```bash - $ make run + make run ``` Alternatively, You can run the operator from your IDE / debugger. @@ -45,13 +45,13 @@ The e2e tests can be executed locally by running the following commands: 3. Set up the test CodeFlare stack: ```bash - $ make setup-e2e + make setup-e2e ``` 4. In a separate terminal, run the e2e suite: ```bash - $ make test-e2e + make test-e2e ``` Alternatively, You can run the e2e test(s) from your IDE / debugger.