DRA kubelet: adapt to v1alpha3 API

This adds the ability to select specific requests inside a claim for a
container.

NodePrepareResources is always called, even if the claim is not used by any
container. This could be useful for drivers where that call has some effect
other than injecting CDI device IDs into containers. It also ensures that
drivers can validate configs.

The pod resource API can no longer report a class for each claim because there
is no such 1:1 relationship anymore. Instead, that API reports claim,
API devices (with driver/pool/device as ID) and CDI device IDs. The kubelet
itself doesn't extract that information from the claim. Instead, it relies on
drivers to report this information when the claim gets prepared. This isolates
the kubelet from API changes.

Because of a faulty E2E test, kubelet was told to contact the wrong driver for
a claim. This was not visible in the kubelet log output. Now changes to the
claim info cache are getting logged. While at it, naming of variables and some
existing log output gets harmonized.

Co-authored-by: Oksana Baranova <oksana.baranova@intel.com>
Co-authored-by: Ed Bartosh <eduard.bartosh@intel.com>

Author: 3 people

pkg/kubelet/apis/podresources/server_v1_test.go

@@ -85,12 +85,14 @@ func TestListPodResourcesV1(t *testing.T) {
 	}
 
 	pluginCDIDevices := []*podresourcesapi.CDIDevice{{Name: "dra-dev0"}, {Name: "dra-dev1"}}
+	draDriverName := "dra.example.com"
+	poolName := "worker-1-pool"
+	deviceName := "gpu-1"
 	draDevs := []*podresourcesapi.DynamicResource{
 		{
-			ClassName:      "resource-class",
 			ClaimName:      "claim-name",
 			ClaimNamespace: "default",
-			ClaimResources: []*podresourcesapi.ClaimResource{{CDIDevices: pluginCDIDevices}},
+			ClaimResources: []*podresourcesapi.ClaimResource{{CDIDevices: pluginCDIDevices, DriverName: draDriverName, PoolName: poolName, DeviceName: deviceName}},
 		},
 	}
 
@@ -893,7 +895,6 @@ func TestGetPodResourcesV1(t *testing.T) {
 	pluginCDIDevices := []*podresourcesapi.CDIDevice{{Name: "dra-dev0"}, {Name: "dra-dev1"}}
 	draDevs := []*podresourcesapi.DynamicResource{
 		{
-			ClassName:      "resource-class",
 			ClaimName:      "claim-name",
 			ClaimNamespace: "default",
 			ClaimResources: []*podresourcesapi.ClaimResource{{CDIDevices: pluginCDIDevices}},

pkg/kubelet/cm/container_manager_linux.go

@@ -661,10 +661,6 @@ func (cm *containerManagerImpl) GetResources(pod *v1.Pod, container *v1.Containe
 		if err != nil {
 			return nil, err
 		}
-		// NOTE: Passing CDI device names as annotations is a temporary solution
-		// It will be removed after all runtimes are updated
-		// to get CDI device names from the ContainerConfig.CDIDevices field
-		opts.Annotations = append(opts.Annotations, resOpts.Annotations...)
 		opts.CDIDevices = append(opts.CDIDevices, resOpts.CDIDevices...)
 	}
 	// Allocate should already be called during predicateAdmitHandler.Admit(),
@@ -965,19 +961,22 @@ func (cm *containerManagerImpl) GetDynamicResources(pod *v1.Pod, container *v1.C
 	}
 	for _, containerClaimInfo := range containerClaimInfos {
 		var claimResources []*podresourcesapi.ClaimResource
-		// TODO: Currently  we maintain a list of ClaimResources, each of which contains
-		// a set of CDIDevices from a different kubelet plugin. In the future we may want to
-		// include the name of the kubelet plugin and/or other types of resources that are
-		// not CDIDevices (assuming the DRAmanager supports this).
-		for _, klPluginCdiDevices := range containerClaimInfo.CDIDevices {
+		for driverName, driverState := range containerClaimInfo.DriverState {
 			var cdiDevices []*podresourcesapi.CDIDevice
-			for _, cdiDevice := range klPluginCdiDevices {
-				cdiDevices = append(cdiDevices, &podresourcesapi.CDIDevice{Name: cdiDevice})
+			for _, device := range driverState.Devices {
+				for _, cdiDeviceID := range device.CDIDeviceIDs {
+					cdiDevices = append(cdiDevices, &podresourcesapi.CDIDevice{Name: cdiDeviceID})
+				}
+				resources := &podresourcesapi.ClaimResource{
+					CDIDevices: cdiDevices,
+					DriverName: driverName,
+					PoolName:   device.PoolName,
+					DeviceName: device.DeviceName,
+				}
+				claimResources = append(claimResources, resources)
 			}
-			claimResources = append(claimResources, &podresourcesapi.ClaimResource{CDIDevices: cdiDevices})
 		}
 		containerDynamicResource := podresourcesapi.DynamicResource{
-			ClassName:      containerClaimInfo.ClassName,
 			ClaimName:      containerClaimInfo.ClaimName,
 			ClaimNamespace: containerClaimInfo.Namespace,
 			ClaimResources: claimResources,

pkg/kubelet/cm/dra/claiminfo.go

@@ -17,14 +17,15 @@ limitations under the License.
 package dra
 
 import (
+	"errors"
 	"fmt"
+	"slices"
 	"sync"
 
 	resourceapi "k8s.io/api/resource/v1alpha3"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/kubernetes/pkg/kubelet/cm/dra/state"
-	"k8s.io/kubernetes/pkg/kubelet/cm/util/cdi"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 )
 
@@ -33,10 +34,7 @@ import (
 // +k8s:deepcopy-gen=true
 type ClaimInfo struct {
 	state.ClaimInfoState
-	// annotations is a mapping of container annotations per DRA plugin associated with
-	// a prepared resource
-	annotations map[string][]kubecontainer.Annotation
-	prepared    bool
+	prepared bool
 }
 
 // claimInfoCache is a cache of processed resource claims keyed by namespace/claimname.
@@ -47,89 +45,45 @@ type claimInfoCache struct {
 }
 
 // newClaimInfoFromClaim creates a new claim info from a resource claim.
-func newClaimInfoFromClaim(claim *resourceapi.ResourceClaim) *ClaimInfo {
-	// Grab the allocation.resourceHandles. If there are no
-	// allocation.resourceHandles, create a single resourceHandle with no
-	// content. This will trigger processing of this claim by a single
-	// kubelet plugin whose name matches resourceClaim.Status.DriverName.
-	resourceHandles := claim.Status.Allocation.ResourceHandles
-	if len(resourceHandles) == 0 {
-		resourceHandles = make([]resourceapi.ResourceHandle, 1)
-	}
+// It verifies that the kubelet can handle the claim.
+func newClaimInfoFromClaim(claim *resourceapi.ResourceClaim) (*ClaimInfo, error) {
 	claimInfoState := state.ClaimInfoState{
-		DriverName:      claim.Status.DriverName,
-		ClassName:       claim.Spec.ResourceClassName,
-		ClaimUID:        claim.UID,
-		ClaimName:       claim.Name,
-		Namespace:       claim.Namespace,
-		PodUIDs:         sets.New[string](),
-		ResourceHandles: resourceHandles,
-		CDIDevices:      make(map[string][]string),
+		ClaimUID:    claim.UID,
+		ClaimName:   claim.Name,
+		Namespace:   claim.Namespace,
+		PodUIDs:     sets.New[string](),
+		DriverState: make(map[string]state.DriverState),
+	}
+	if claim.Status.Allocation == nil {
+		return nil, errors.New("not allocated")
+	}
+	for _, result := range claim.Status.Allocation.Devices.Results {
+		claimInfoState.DriverState[result.Driver] = state.DriverState{}
 	}
 	info := &ClaimInfo{
 		ClaimInfoState: claimInfoState,
-		annotations:    make(map[string][]kubecontainer.Annotation),
 		prepared:       false,
 	}
-	return info
+	return info, nil
 }
 
 // newClaimInfoFromClaim creates a new claim info from a checkpointed claim info state object.
 func newClaimInfoFromState(state *state.ClaimInfoState) *ClaimInfo {
 	info := &ClaimInfo{
 		ClaimInfoState: *state.DeepCopy(),
-		annotations:    make(map[string][]kubecontainer.Annotation),
 		prepared:       false,
 	}
-	for pluginName, devices := range info.CDIDevices {
-		annotations, _ := cdi.GenerateAnnotations(info.ClaimUID, info.DriverName, devices)
-		info.annotations[pluginName] = append(info.annotations[pluginName], annotations...)
-	}
 	return info
 }
 
 // setCDIDevices adds a set of CDI devices to the claim info.
-func (info *ClaimInfo) setCDIDevices(pluginName string, cdiDevices []string) error {
-	// NOTE: Passing CDI device names as annotations is a temporary solution
-	// It will be removed after all runtimes are updated
-	// to get CDI device names from the ContainerConfig.CDIDevices field
-	annotations, err := cdi.GenerateAnnotations(info.ClaimUID, info.DriverName, cdiDevices)
-	if err != nil {
-		return fmt.Errorf("failed to generate container annotations, err: %+v", err)
-	}
-
-	if info.CDIDevices == nil {
-		info.CDIDevices = make(map[string][]string)
-	}
-
-	if info.annotations == nil {
-		info.annotations = make(map[string][]kubecontainer.Annotation)
-	}
-
-	info.CDIDevices[pluginName] = cdiDevices
-	info.annotations[pluginName] = annotations
-
-	return nil
-}
-
-// annotationsAsList returns container annotations as a single list.
-func (info *ClaimInfo) annotationsAsList() []kubecontainer.Annotation {
-	var lst []kubecontainer.Annotation
-	for _, v := range info.annotations {
-		lst = append(lst, v...)
+func (info *ClaimInfo) addDevice(driverName string, device state.Device) {
+	if info.DriverState == nil {
+		info.DriverState = make(map[string]state.DriverState)
 	}
-	return lst
-}
-
-// cdiDevicesAsList returns a list of CDIDevices from the provided claim info.
-func (info *ClaimInfo) cdiDevicesAsList() []kubecontainer.CDIDevice {
-	var cdiDevices []kubecontainer.CDIDevice
-	for _, devices := range info.CDIDevices {
-		for _, device := range devices {
-			cdiDevices = append(cdiDevices, kubecontainer.CDIDevice{Name: device})
-		}
-	}
-	return cdiDevices
+	driverState := info.DriverState[driverName]
+	driverState.Devices = append(driverState.Devices, device)
+	info.DriverState[driverName] = driverState
 }
 
 // addPodReference adds a pod reference to the claim info.
@@ -240,3 +194,20 @@ func (cache *claimInfoCache) syncToCheckpoint() error {
 	}
 	return cache.state.Store(claimInfoStateList)
 }
+
+// cdiDevicesAsList returns a list of CDIDevices from the provided claim info.
+// When the request name is non-empty, only devices relevant for that request
+// are returned.
+func (info *ClaimInfo) cdiDevicesAsList(requestName string) []kubecontainer.CDIDevice {
+	var cdiDevices []kubecontainer.CDIDevice
+	for _, driverData := range info.DriverState {
+		for _, device := range driverData.Devices {
+			if requestName == "" || len(device.RequestNames) == 0 || slices.Contains(device.RequestNames, requestName) {
+				for _, cdiDeviceID := range device.CDIDeviceIDs {
+					cdiDevices = append(cdiDevices, kubecontainer.CDIDevice{Name: cdiDeviceID})
+				}
+			}
+		}
+	}
+	return cdiDevices
+}