DRA: update helper packages

Publishing ResourceSlices now supports network-attached devices and the new
v1alpha3 API.  The logic for splitting up across different slices is missing.

Author: pohly

staging/src/k8s.io/dynamic-resource-allocation/controller/controller.go

@@ -62,20 +62,6 @@ type Controller interface {
 
 // Driver provides the actual allocation and deallocation operations.
 type Driver interface {
-	// GetClassParameters is called to retrieve the parameter object
-	// referenced by a class. The content should be validated now if
-	// possible. class.Parameters may be nil.
-	//
-	// The caller wraps the error to include the parameter reference.
-	GetClassParameters(ctx context.Context, class *resourceapi.ResourceClass) (interface{}, error)
-
-	// GetClaimParameters is called to retrieve the parameter object
-	// referenced by a claim. The content should be validated now if
-	// possible. claim.Spec.Parameters may be nil.
-	//
-	// The caller wraps the error to include the parameter reference.
-	GetClaimParameters(ctx context.Context, claim *resourceapi.ResourceClaim, class *resourceapi.ResourceClass, classParameters interface{}) (interface{}, error)
-
 	// Allocate is called when all same-driver ResourceClaims for Pod are ready
 	// to be allocated. The selectedNode is empty for ResourceClaims with immediate
 	// allocation, in which case the resource driver decides itself where
@@ -136,11 +122,9 @@ type Driver interface {
 // ClaimAllocation represents information about one particular
 // pod.Spec.ResourceClaim entry.
 type ClaimAllocation struct {
-	PodClaimName    string
-	Claim           *resourceapi.ResourceClaim
-	Class           *resourceapi.ResourceClass
-	ClaimParameters interface{}
-	ClassParameters interface{}
+	PodClaimName  string
+	Claim         *resourceapi.ResourceClaim
+	DeviceClasses map[string]*resourceapi.DeviceClass
 
 	// UnsuitableNodes needs to be filled in by the driver when
 	// Driver.UnsuitableNodes gets called.
@@ -162,15 +146,12 @@ type controller struct {
 	driver              Driver
 	setReservedFor      bool
 	kubeClient          kubernetes.Interface
-	claimNameLookup     *resourceclaim.Lookup
 	queue               workqueue.TypedRateLimitingInterface[string]
 	eventRecorder       record.EventRecorder
-	rcLister            resourcelisters.ResourceClassLister
-	rcSynced            cache.InformerSynced
+	dcLister            resourcelisters.DeviceClassLister
 	claimCache          cache.MutationCache
 	schedulingCtxLister resourcelisters.PodSchedulingContextLister
-	claimSynced         cache.InformerSynced
-	schedulingCtxSynced cache.InformerSynced
+	synced              []cache.InformerSynced
 }
 
 // TODO: make it configurable
@@ -184,10 +165,9 @@ func New(
 	kubeClient kubernetes.Interface,
 	informerFactory informers.SharedInformerFactory) Controller {
 	logger := klog.LoggerWithName(klog.FromContext(ctx), "resource controller")
-	rcInformer := informerFactory.Resource().V1alpha3().ResourceClasses()
+	dcInformer := informerFactory.Resource().V1alpha3().DeviceClasses()
 	claimInformer := informerFactory.Resource().V1alpha3().ResourceClaims()
 	schedulingCtxInformer := informerFactory.Resource().V1alpha3().PodSchedulingContexts()
-	claimNameLookup := resourceclaim.NewNameLookup(kubeClient)
 
 	eventBroadcaster := record.NewBroadcaster(record.WithContext(ctx))
 	go func() {
@@ -228,15 +208,16 @@ func New(
 		driver:              driver,
 		setReservedFor:      true,
 		kubeClient:          kubeClient,
-		claimNameLookup:     claimNameLookup,
-		rcLister:            rcInformer.Lister(),
-		rcSynced:            rcInformer.Informer().HasSynced,
+		dcLister:            dcInformer.Lister(),
 		claimCache:          claimCache,
-		claimSynced:         claimInformer.Informer().HasSynced,
 		schedulingCtxLister: schedulingCtxInformer.Lister(),
-		schedulingCtxSynced: schedulingCtxInformer.Informer().HasSynced,
 		queue:               queue,
 		eventRecorder:       eventRecorder,
+		synced: []cache.InformerSynced{
+			dcInformer.Informer().HasSynced,
+			claimInformer.Informer().HasSynced,
+			schedulingCtxInformer.Informer().HasSynced,
+		},
 	}
 
 	loggerV6 := logger.V(6)
@@ -341,7 +322,7 @@ func (ctrl *controller) Run(workers int) {
 
 	stopCh := ctrl.ctx.Done()
 
-	if !cache.WaitForCacheSync(stopCh, ctrl.rcSynced, ctrl.claimSynced, ctrl.schedulingCtxSynced) {
+	if !cache.WaitForCacheSync(stopCh, ctrl.synced...) {
 		ctrl.logger.Error(nil, "Cannot sync caches")
 		return
 	}
@@ -471,20 +452,19 @@ func (ctrl *controller) syncClaim(ctx context.Context, claim *resourceapi.Resour
 			if claim.Status.Allocation != nil {
 				// Allocation was completed. Deallocate before proceeding.
 				if err := ctrl.driver.Deallocate(ctx, claim); err != nil {
-					return fmt.Errorf("deallocate: %v", err)
+					return fmt.Errorf("deallocate: %w", err)
 				}
 				claim.Status.Allocation = nil
-				claim.Status.DriverName = ""
 				claim.Status.DeallocationRequested = false
 				claim, err = ctrl.kubeClient.ResourceV1alpha3().ResourceClaims(claim.Namespace).UpdateStatus(ctx, claim, metav1.UpdateOptions{})
 				if err != nil {
-					return fmt.Errorf("remove allocation: %v", err)
+					return fmt.Errorf("remove allocation: %w", err)
 				}
 				ctrl.claimCache.Mutation(claim)
 			} else {
 				// Ensure that there is no on-going allocation.
 				if err := ctrl.driver.Deallocate(ctx, claim); err != nil {
-					return fmt.Errorf("stop allocation: %v", err)
+					return fmt.Errorf("stop allocation: %w", err)
 				}
 			}
 
@@ -493,15 +473,15 @@ func (ctrl *controller) syncClaim(ctx context.Context, claim *resourceapi.Resour
 				claim.Status.DeallocationRequested = false
 				claim, err = ctrl.kubeClient.ResourceV1alpha3().ResourceClaims(claim.Namespace).UpdateStatus(ctx, claim, metav1.UpdateOptions{})
 				if err != nil {
-					return fmt.Errorf("remove deallocation: %v", err)
+					return fmt.Errorf("remove deallocation: %w", err)
 				}
 				ctrl.claimCache.Mutation(claim)
 			}
 
 			claim.Finalizers = ctrl.removeFinalizer(claim.Finalizers)
 			claim, err = ctrl.kubeClient.ResourceV1alpha3().ResourceClaims(claim.Namespace).Update(ctx, claim, metav1.UpdateOptions{})
 			if err != nil {
-				return fmt.Errorf("remove finalizer: %v", err)
+				return fmt.Errorf("remove finalizer: %w", err)
 			}
 			ctrl.claimCache.Mutation(claim)
 		}
@@ -519,24 +499,6 @@ func (ctrl *controller) syncClaim(ctx context.Context, claim *resourceapi.Resour
 	return nil
 }
 
-func (ctrl *controller) getParameters(ctx context.Context, claim *resourceapi.ResourceClaim, class *resourceapi.ResourceClass, notifyClaim bool) (claimParameters, classParameters interface{}, err error) {
-	classParameters, err = ctrl.driver.GetClassParameters(ctx, class)
-	if err != nil {
-		ctrl.eventRecorder.Event(class, v1.EventTypeWarning, "Failed", err.Error())
-		err = fmt.Errorf("class parameters %s: %v", class.ParametersRef, err)
-		return
-	}
-	claimParameters, err = ctrl.driver.GetClaimParameters(ctx, claim, class, classParameters)
-	if err != nil {
-		if notifyClaim {
-			ctrl.eventRecorder.Event(claim, v1.EventTypeWarning, "Failed", err.Error())
-		}
-		err = fmt.Errorf("claim parameters %s: %v", claim.Spec.ParametersRef, err)
-		return
-	}
-	return
-}
-
 // allocateClaims filters list of claims, keeps those needing allocation and asks driver to do the allocations.
 // Driver is supposed to write the AllocationResult and Error field into argument claims slice.
 func (ctrl *controller) allocateClaims(ctx context.Context, claims []*ClaimAllocation, selectedNode string, selectedUser *resourceapi.ResourceClaimConsumerReference) {
@@ -572,7 +534,7 @@ func (ctrl *controller) allocateClaims(ctx context.Context, claims []*ClaimAlloc
 			claim, err = ctrl.kubeClient.ResourceV1alpha3().ResourceClaims(claim.Namespace).Update(ctx, claim, metav1.UpdateOptions{})
 			if err != nil {
 				logger.Error(err, "add finalizer", "claim", claim.Name)
-				claimAllocation.Error = fmt.Errorf("add finalizer: %v", err)
+				claimAllocation.Error = fmt.Errorf("add finalizer: %w", err)
 				// Do not save claim to ask for Allocate from Driver.
 				continue
 			}
@@ -602,14 +564,14 @@ func (ctrl *controller) allocateClaims(ctx context.Context, claims []*ClaimAlloc
 		logger.V(5).Info("successfully allocated", "claim", klog.KObj(claimAllocation.Claim))
 		claim := claimAllocation.Claim.DeepCopy()
 		claim.Status.Allocation = claimAllocation.Allocation
-		claim.Status.DriverName = ctrl.name
+		claim.Status.Allocation.Controller = ctrl.name
 		if selectedUser != nil && ctrl.setReservedFor {
 			claim.Status.ReservedFor = append(claim.Status.ReservedFor, *selectedUser)
 		}
 		logger.V(6).Info("Updating claim after allocation", "claim", claim)
 		claim, err := ctrl.kubeClient.ResourceV1alpha3().ResourceClaims(claim.Namespace).UpdateStatus(ctx, claim, metav1.UpdateOptions{})
 		if err != nil {
-			claimAllocation.Error = fmt.Errorf("add allocation: %v", err)
+			claimAllocation.Error = fmt.Errorf("add allocation: %w", err)
 			continue
 		}
 
@@ -619,7 +581,7 @@ func (ctrl *controller) allocateClaims(ctx context.Context, claims []*ClaimAlloc
 }
 
 func (ctrl *controller) checkPodClaim(ctx context.Context, pod *v1.Pod, podClaim v1.PodResourceClaim) (*ClaimAllocation, error) {
-	claimName, mustCheckOwner, err := ctrl.claimNameLookup.Name(pod, &podClaim)
+	claimName, mustCheckOwner, err := resourceclaim.Name(pod, &podClaim)
 	if err != nil {
 		return nil, err
 	}
@@ -642,26 +604,30 @@ func (ctrl *controller) checkPodClaim(ctx context.Context, pod *v1.Pod, podClaim
 		// need to be done for the claim either.
 		return nil, nil
 	}
-	class, err := ctrl.rcLister.Get(claim.Spec.ResourceClassName)
-	if err != nil {
-		return nil, err
-	}
-	if class.DriverName != ctrl.name {
+	if claim.Spec.Controller != ctrl.name {
 		return nil, nil
 	}
-	// Check parameters. Record event to claim and pod if parameters are invalid.
-	claimParameters, classParameters, err := ctrl.getParameters(ctx, claim, class, true)
-	if err != nil {
-		ctrl.eventRecorder.Event(pod, v1.EventTypeWarning, "Failed", fmt.Sprintf("claim %v: %v", claim.Name, err.Error()))
-		return nil, err
+
+	// Sanity checks and preparations...
+	ca := &ClaimAllocation{
+		PodClaimName:  podClaim.Name,
+		Claim:         claim,
+		DeviceClasses: make(map[string]*resourceapi.DeviceClass),
+	}
+	for _, request := range claim.Spec.Devices.Requests {
+		if request.DeviceClassName == "" {
+			// Some unknown request. Abort!
+			return nil, fmt.Errorf("claim %s: unknown request type in request %s", klog.KObj(claim), request.Name)
+		}
+		deviceClassName := request.DeviceClassName
+		class, err := ctrl.dcLister.Get(deviceClassName)
+		if err != nil {
+			return nil, fmt.Errorf("claim %s: request %s: class %s: %w", klog.KObj(claim), request.Name, deviceClassName, err)
+		}
+		ca.DeviceClasses[deviceClassName] = class
 	}
-	return &ClaimAllocation{
-		PodClaimName:    podClaim.Name,
-		Claim:           claim,
-		Class:           class,
-		ClaimParameters: claimParameters,
-		ClassParameters: classParameters,
-	}, nil
+
+	return ca, nil
 }
 
 // syncPodSchedulingContext determines which next action may be needed for a PodSchedulingContext object
@@ -709,7 +675,7 @@ func (ctrl *controller) syncPodSchedulingContexts(ctx context.Context, schedulin
 	for _, podClaim := range pod.Spec.ResourceClaims {
 		delayed, err := ctrl.checkPodClaim(ctx, pod, podClaim)
 		if err != nil {
-			return fmt.Errorf("pod claim %s: %v", podClaim.Name, err)
+			return fmt.Errorf("pod claim %s: %w", podClaim.Name, err)
 		}
 		if delayed == nil {
 			// Nothing to do for it. This can change, so keep checking.
@@ -739,7 +705,7 @@ func (ctrl *controller) syncPodSchedulingContexts(ctx context.Context, schedulin
 	}
 	if len(schedulingCtx.Spec.PotentialNodes) > 0 {
 		if err := ctrl.driver.UnsuitableNodes(ctx, pod, claims, potentialNodes); err != nil {
-			return fmt.Errorf("checking potential nodes: %v", err)
+			return fmt.Errorf("checking potential nodes: %w", err)
 		}
 	}
 	logger.V(5).Info("pending pod claims", "claims", claims, "selectedNode", selectedNode)
@@ -772,7 +738,7 @@ func (ctrl *controller) syncPodSchedulingContexts(ctx context.Context, schedulin
 						allErrors = append(allErrors, delayed.Error)
 					} else {
 						// Include claim name, it's not in the underlying error.
-						allErrors = append(allErrors, fmt.Errorf("claim %s: %v", delayed.Claim.Name, delayed.Error))
+						allErrors = append(allErrors, fmt.Errorf("claim %s: %w", delayed.Claim.Name, delayed.Error))
 					}
 				}
 			}
@@ -807,7 +773,7 @@ func (ctrl *controller) syncPodSchedulingContexts(ctx context.Context, schedulin
 	if modified {
 		logger.V(6).Info("Updating pod scheduling with modified unsuitable nodes", "podSchedulingCtx", schedulingCtx)
 		if _, err := ctrl.kubeClient.ResourceV1alpha3().PodSchedulingContexts(schedulingCtx.Namespace).UpdateStatus(ctx, schedulingCtx, metav1.UpdateOptions{}); err != nil {
-			return fmt.Errorf("update unsuitable node status: %v", err)
+			return fmt.Errorf("update unsuitable node status: %w", err)
 		}
 	}