Update DRA testing to stable API version and prepare it to test more types of drivers

Author: emerbe

clusterloader2/pkg/dependency/dra/dra.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"time"
 
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/klog/v2"
@@ -31,16 +32,16 @@ import (
 )
 
 const (
-	draDependencyName = "DRATestDriver"
-	//TODO: this needs to be converted into a parameter. Will will not need this until parititionable devices test
+	draDependencyName      = "DRATestDriver"
 	draNamespace           = "dra-example-driver"
+	draManifests           = "manifests/*.yaml"
 	defaultWorkerNodeCount = "100"
 	draDaemonsetName       = "dra-example-driver-kubeletplugin"
 	checkDRAReadyInterval  = 30 * time.Second
 	defaultDRATimeout      = 10 * time.Minute
 )
 
-//go:embed manifests/*.yaml
+//go:embed manifests/**/*.yaml
 var manifestsFS embed.FS
 
 func init() {
@@ -57,13 +58,24 @@ type draDependency struct{}
 
 func (d *draDependency) Setup(config *dependency.Config) error {
 	klog.V(2).Infof("%s: Installing DRA example driver", d)
-	if err := client.CreateNamespace(config.ClusterFramework.GetClientSets().GetClient(), draNamespace); err != nil {
-		return fmt.Errorf("namespace %s creation error: %v", draNamespace, err)
+
+	namespace, err := getNamespace(config)
+	if err != nil {
+		return err
 	}
 
-	namespace, ok := config.Params["Namespace"]
-	if !ok {
-		namespace = draNamespace
+	if err := client.CreateNamespace(config.ClusterFramework.GetClientSets().GetClient(), namespace); err != nil {
+		return fmt.Errorf("namespace %s creation error: %v", namespace, err)
+	}
+
+	manifests, err := getManifests(config)
+	if err != nil {
+		return err
+	}
+
+	daemonsetName, err := getDaemonset(config)
+	if err != nil {
+		return err
 	}
 
 	mapping := map[string]interface{}{
@@ -72,7 +84,7 @@ func (d *draDependency) Setup(config *dependency.Config) error {
 	}
 	if err := config.ClusterFramework.ApplyTemplatedManifests(
 		manifestsFS,
-		"manifests/*.yaml",
+		manifests,
 		mapping,
 		client.Retry(client.IsRetryableAPIError),
 	); err != nil {
@@ -82,8 +94,8 @@ func (d *draDependency) Setup(config *dependency.Config) error {
 	if err != nil {
 		return err
 	}
-	klog.V(2).Infof("%s: checking if DRA driver %s is healthy", d, draDaemonsetName)
-	if err := d.waitForDRADriverToBeHealthy(config, timeout); err != nil {
+	klog.V(2).Infof("%s: checking if DRA driver %s is healthy", d, daemonsetName)
+	if err := d.waitForDRADriverToBeHealthy(config, timeout, daemonsetName, namespace); err != nil {
 		return err
 	}
 
@@ -94,25 +106,30 @@ func (d *draDependency) Setup(config *dependency.Config) error {
 func (d *draDependency) Teardown(config *dependency.Config) error {
 	klog.V(2).Infof("%s: Tearing down DRA example driver", d)
 
+	namespace, err := getNamespace(config)
+	if err != nil {
+		return err
+	}
+
 	// Delete namespace (this will delete all resources in it)
-	if err := client.DeleteNamespace(config.ClusterFramework.GetClientSets().GetClient(), draNamespace); err != nil {
-		return fmt.Errorf("deleting %s namespace error: %v", draNamespace, err)
+	if err := client.DeleteNamespace(config.ClusterFramework.GetClientSets().GetClient(), namespace); err != nil {
+		return fmt.Errorf("deleting %s namespace error: %v", namespace, err)
 	}
 
-	if err := client.WaitForDeleteNamespace(config.ClusterFramework.GetClientSets().GetClient(), draNamespace, client.DefaultNamespaceDeletionTimeout); err != nil {
+	if err := client.WaitForDeleteNamespace(config.ClusterFramework.GetClientSets().GetClient(), namespace, client.DefaultNamespaceDeletionTimeout); err != nil {
 		return err
 	}
 
 	klog.V(2).Infof("%s: DRA example driver uninstalled successfully", d)
 	return nil
 }
 
-func (d *draDependency) waitForDRADriverToBeHealthy(config *dependency.Config, timeout time.Duration) error {
+func (d *draDependency) waitForDRADriverToBeHealthy(config *dependency.Config, timeout time.Duration, daemonsetName string, namespace string) error {
 	if err := wait.PollImmediate(
 		checkDRAReadyInterval,
 		timeout,
 		func() (done bool, err error) {
-			return d.isDRADriverReady(config)
+			return d.isDRADriverReady(config, daemonsetName, namespace)
 		}); err != nil {
 		return err
 	}
@@ -127,27 +144,33 @@ func (d *draDependency) waitForDRADriverToBeHealthy(config *dependency.Config, t
 	return nil
 }
 
-func (d *draDependency) isDRADriverReady(config *dependency.Config) (done bool, err error) {
+func (d *draDependency) isDRADriverReady(config *dependency.Config, daemonsetName string, namespace string) (done bool, err error) {
 	ds, err := config.ClusterFramework.GetClientSets().
 		GetClient().
 		AppsV1().
-		DaemonSets(draNamespace).
-		Get(context.Background(), draDaemonsetName, metav1.GetOptions{})
+		DaemonSets(namespace).
+		Get(context.Background(), daemonsetName, metav1.GetOptions{})
 	if err != nil {
-		return false, fmt.Errorf("failed to get %s: %v", draDaemonsetName, err)
+		return false, fmt.Errorf("failed to get %s: %v", daemonsetName, err)
 	}
 	ready := ds.Status.NumberReady == ds.Status.DesiredNumberScheduled
 	if !ready {
 		klog.V(2).Infof("%s is not ready, "+
-			"DesiredNumberScheduled: %d, NumberReady: %d", draDaemonsetName, ds.Status.DesiredNumberScheduled, ds.Status.NumberReady)
+			"DesiredNumberScheduled: %d, NumberReady: %d", daemonsetName, ds.Status.DesiredNumberScheduled, ds.Status.NumberReady)
 	}
 	return ready, nil
 }
 
 func isResourceSlicesPublished(config *dependency.Config) (bool, error) {
-	workerCount := int(getWorkerCount(config).(float64))
+	// Get a list of all nodes
+	nodes, err := getReadyNodesCount(config)
+	if err != nil {
+		return false, fmt.Errorf("failed to list nodes: %v", err)
+	}
+
+	workerCount := nodes
 
-	resourceSlices, err := config.ClusterFramework.GetClientSets().GetClient().ResourceV1beta1().ResourceSlices().List(context.Background(), metav1.ListOptions{})
+	resourceSlices, err := config.ClusterFramework.GetClientSets().GetClient().ResourceV1().ResourceSlices().List(context.Background(), metav1.ListOptions{})
 	if err != nil {
 		return false, fmt.Errorf("failed to list resourceslices: %v", err)
 	}
@@ -159,6 +182,25 @@ func isResourceSlicesPublished(config *dependency.Config) (bool, error) {
 	return true, nil
 }
 
+func getReadyNodesCount(config *dependency.Config) (int, error) {
+	// Get a list of all nodes
+	nodes, err := config.ClusterFramework.GetClientSets().GetClient().CoreV1().Nodes().List(context.Background(), metav1.ListOptions{})
+	if err != nil {
+		return 0, fmt.Errorf("failed to list nodes: %v", err)
+	}
+
+	readyNodes := 0
+	for _, node := range nodes.Items {
+		for _, condition := range node.Status.Conditions {
+			if condition.Type == corev1.NodeReady && condition.Status == corev1.ConditionTrue {
+				readyNodes++
+				break // Found the Ready condition, move to the next node
+			}
+		}
+	}
+	return readyNodes, nil
+}
+
 func getWorkerCount(config *dependency.Config) interface{} {
 	workerCount, ok := config.Params["WorkerNodeCount"]
 	if !ok {
@@ -167,6 +209,43 @@ func getWorkerCount(config *dependency.Config) interface{} {
 	return workerCount
 }
 
+func getNamespace(config *dependency.Config) (string, error) {
+	namespace, ok := config.Params["Namespace"]
+	if !ok {
+		namespace = draNamespace
+	}
+	namespaceString, ok := namespace.(string)
+
+	if !ok {
+		return "", fmt.Errorf("namespace parameter is not a string: %v", namespace)
+	}
+	return namespaceString, nil
+}
+
+func getManifests(config *dependency.Config) (string, error) {
+	manifests, ok := config.Params["Manifests"]
+	if !ok {
+		manifests = draManifests
+	}
+	manifestsString, ok := manifests.(string)
+	if !ok {
+		return "", fmt.Errorf("manifests parameter is not a string: %v", manifests)
+	}
+	return "manifests/" + manifestsString + "/*.yaml", nil
+}
+
+func getDaemonset(config *dependency.Config) (string, error) {
+	daemonsetName, ok := config.Params["DaemonsetName"]
+	if !ok {
+		daemonsetName = draDaemonsetName
+	}
+	daemonsetNameString, ok := daemonsetName.(string)
+	if !ok {
+		return "", fmt.Errorf("DaemonsetName parameter is not a string: %v", daemonsetName)
+	}
+	return daemonsetNameString, nil
+}
+
 // String returns string representation of this dependency.
 func (d *draDependency) String() string {
 	return draDependencyName

clusterloader2/pkg/dependency/dra/manifests/clusterrole.yaml renamed to clusterloader2/pkg/dependency/dra/manifests/example/clusterrole.yaml

@@ -1,6 +1,6 @@
 ---
 # Source: dra-example-driver/templates/deviceclass.yaml
-apiVersion: resource.k8s.io/v1beta1
+apiVersion: resource.k8s.io/v1
 kind: DeviceClass
 metadata:
   name: gpu.example.com

clusterloader2/pkg/dependency/dra/manifests/clusterrolebinding.yaml renamed to clusterloader2/pkg/dependency/dra/manifests/example/clusterrolebinding.yaml

@@ -34,7 +34,7 @@ spec:
       - name: plugin
         securityContext:
           privileged: true
-        image: registry.k8s.io/dra-example-driver/dra-example-driver:v0.1.0
+        image: registry.k8s.io/dra-example-driver/dra-example-driver:v0.2.0
         imagePullPolicy: IfNotPresent
         command: ["dra-example-kubeletplugin"]
         resources:

clusterloader2/pkg/dependency/dra/manifests/deviceclass.yaml renamed to clusterloader2/pkg/dependency/dra/manifests/example/deviceclass.yaml

@@ -9,7 +9,7 @@ spec:
   matchConstraints:
     resourceRules:
     - apiGroups:   ["resource.k8s.io"]
-      apiVersions: ["v1beta1"]
+      apiVersions: ["v1"]
       operations:  ["CREATE", "UPDATE", "DELETE"]
       resources:   ["resourceslices"]
   matchConditions:

clusterloader2/pkg/dependency/dra/manifests/kubeletplugin.yaml renamed to clusterloader2/pkg/dependency/dra/manifests/example/kubeletplugin.yaml

@@ -24,7 +24,7 @@ import (
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
-	resourcev1beta2 "k8s.io/api/resource/v1beta2"
+	resourcev1 "k8s.io/api/resource/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/watch"
@@ -145,18 +145,16 @@ func (m *resourceClaimAllocationLatencyMeasurement) start(c clientset.Interface)
 	m.isRunning = true
 	m.stopCh = make(chan struct{})
 	m.client = c
-
 	lw := &cache.ListWatch{
 		ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
 			m.selector.ApplySelectors(&options)
-			return c.ResourceV1beta2().ResourceClaims(m.selector.Namespace).List(context.TODO(), options)
+			return c.ResourceV1().ResourceClaims(m.selector.Namespace).List(context.TODO(), options)
 		},
 		WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
 			m.selector.ApplySelectors(&options)
-			return c.ResourceV1beta2().ResourceClaims(m.selector.Namespace).Watch(context.TODO(), options)
+			return c.ResourceV1().ResourceClaims(m.selector.Namespace).Watch(context.TODO(), options)
 		},
 	}
-
 	claimInf := informer.NewInformer(lw, m.addEvent)
 
 	podLW := &cache.ListWatch{
@@ -219,7 +217,7 @@ func (m *resourceClaimAllocationLatencyMeasurement) processEvent(ev *claimEventD
 		return
 	}
 
-	claim, ok := ev.obj.(*resourcev1beta2.ResourceClaim)
+	claim, ok := ev.obj.(*resourcev1.ResourceClaim)
 	if !ok {
 		return
 	}
@@ -303,7 +301,7 @@ func (m *resourceClaimAllocationLatencyMeasurement) gather(_ clientset.Interface
 	return []measurement.Summary{measurement.CreateSummary(summaryName, "json", content)}, err
 }
 
-func isAllocated(claim *resourcev1beta2.ResourceClaim) bool {
+func isAllocated(claim *resourcev1.ResourceClaim) bool {
 	return claim.Status.Allocation != nil || len(claim.Status.ReservedFor) > 0 || len(claim.Status.Devices) > 0
 }
 
@@ -316,7 +314,7 @@ func usesResourceClaimTemplate(p *corev1.Pod) bool {
 	return false
 }
 
-func (m *resourceClaimAllocationLatencyMeasurement) getCachedPodCreateTime(cl *resourcev1beta2.ResourceClaim) (time.Time, bool) {
+func (m *resourceClaimAllocationLatencyMeasurement) getCachedPodCreateTime(cl *resourcev1.ResourceClaim) (time.Time, bool) {
 	for _, o := range cl.OwnerReferences {
 		if o.Kind == "Pod" && o.Name != "" {
 			key := fmt.Sprintf("%s/%s", cl.Namespace, o.Name)
@@ -329,7 +327,7 @@ func (m *resourceClaimAllocationLatencyMeasurement) getCachedPodCreateTime(cl *r
 	return time.Time{}, false
 }
 
-func (m *resourceClaimAllocationLatencyMeasurement) fetchPodCreateTime(cl *resourcev1beta2.ResourceClaim) (time.Time, bool) {
+func (m *resourceClaimAllocationLatencyMeasurement) fetchPodCreateTime(cl *resourcev1.ResourceClaim) (time.Time, bool) {
 	for _, o := range cl.OwnerReferences {
 		if o.Kind == "Pod" && o.Name != "" {
 			atomic.AddInt64(&m.podGetCalls, 1)