Handle multiple GPUs in CDI spec generation from CSV

This change allows CDI specs to be generated for multiple
devices when using CSV mode. This can be used in cases where
a Tegra-based system consists of an iGPU and dGPU.

This behavior can be opted out of using the disable-multiple-csv-devices
feature flag. This can be specified by adding the

	--feaure-flags=disable-multiple-csv-devices

command line option to the nvidia-ctk cdi generate command or to the
automatic CDI spec generation by adding

NVIDIA_CTK_CDI_GENERATE_FEATURE_FLAGS=disable-multiple-csv-devices

to the /etc/nvidia-container-toolkit/nvidia-cdi-refresh.env file.

Signed-off-by: Evan Lezar <[email protected]>

Author: elezar

pkg/nvcdi/api.go

@@ -88,4 +88,8 @@ const (
 	// FeatureEnableCoherentAnnotations enables the addition of annotations
 	// coherent or non-coherent devices.
 	FeatureEnableCoherentAnnotations = FeatureFlag("enable-coherent-annotations")
+
+	// FeatureDisableMultipleCSVDevices disables the handling of multiple devices
+	// in CSV mode.
+	FeatureDisableMultipleCSVDevices = FeatureFlag("disable-multiple-csv-devices")
 )

pkg/nvcdi/common-nvml.go

@@ -25,16 +25,7 @@ import (
 // newCommonNVMLDiscoverer returns a discoverer for entities that are not associated with a specific CDI device.
 // This includes driver libraries and meta devices, for example.
 func (l *nvmllib) newCommonNVMLDiscoverer() (discover.Discover, error) {
-	metaDevices := discover.NewCharDeviceDiscoverer(
-		l.logger,
-		l.devRoot,
-		[]string{
-			"/dev/nvidia-modeset",
-			"/dev/nvidia-uvm-tools",
-			"/dev/nvidia-uvm",
-			"/dev/nvidiactl",
-		},
-	)
+	metaDevices := l.controlDeviceNodeDiscoverer()
 
 	graphicsMounts, err := discover.NewGraphicsMountsDiscoverer(l.logger, l.driver, l.hookCreator)
 	if err != nil {
@@ -54,3 +45,16 @@ func (l *nvmllib) newCommonNVMLDiscoverer() (discover.Discover, error) {
 
 	return d, nil
 }
+
+func (l *nvmllib) controlDeviceNodeDiscoverer() discover.Discover {
+	return discover.NewCharDeviceDiscoverer(
+		l.logger,
+		l.devRoot,
+		[]string{
+			"/dev/nvidia-modeset",
+			"/dev/nvidia-uvm-tools",
+			"/dev/nvidia-uvm",
+			"/dev/nvidiactl",
+		},
+	)
+}

pkg/nvcdi/full-gpu-nvml.go

@@ -37,7 +37,8 @@ type fullGPUDeviceSpecGenerator struct {
 	uuid  string
 	index int
 
-	featureFlags map[FeatureFlag]bool
+	featureFlags          map[FeatureFlag]bool
+	additionalDiscoverers []discover.Discover
 }
 
 var _ DeviceSpecGenerator = (*fullGPUDeviceSpecGenerator)(nil)
@@ -145,7 +146,6 @@ func (l *fullGPUDeviceSpecGenerator) getDeviceEdits() (*cdi.ContainerEdits, erro
 	if err != nil {
 		return nil, fmt.Errorf("failed to create device discoverer: %v", err)
 	}
-
 	editsForDevice, err := edits.FromDiscoverer(deviceDiscoverer)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create container edits for device: %v", err)
@@ -177,10 +177,18 @@ func (l *fullGPUDeviceSpecGenerator) newFullGPUDiscoverer(d device.Device) (disc
 		deviceNodes,
 	)
 
-	dd := discover.Merge(
+	var discoverers []discover.Discover
+
+	discoverers = append(discoverers,
 		deviceNodes,
 		deviceFolderPermissionHooks,
 	)
 
+	discoverers = append(discoverers, l.additionalDiscoverers...)
+
+	dd := discover.Merge(
+		discoverers...,
+	)
+
 	return dd, nil
 }