nvsandboxutils: Add usage of GetGpuResource and GetFileContent APIs

This change adds a new discoverer for Sandboxutils to report the file system paths and associated symbolic links using GetGpuResource and GetFileContent APIs. Both GPU and MIG devices are supported. If the Sandboxutils discoverer fails, the NVML discoverer is used to report the file system information. Signed-off-by: Evan Lezar <[email protected]> Signed-off-by: Huy Nguyen <[email protected]> Signed-off-by: Sananya Majumder <[email protected]>
NVIDIA · Aug 6, 2024 · 4a6f079 · 4a6f079
1 parent 93edec3
commit 4a6f079
Show file tree

Hide file tree

Showing 9 changed files with 537 additions and 3 deletions.
diff --git a/.golangci.yml b/.golangci.yml
@@ -34,3 +34,8 @@ issues:
     linters:
     - errcheck
     text: config.Delete
+  # RENDERD refers to the Render Device and not the past tense of render.
+  - path: .*.go
+    linters:
+    - misspell
+    text: "`RENDERD` is a misspelling of `RENDERED`"
diff --git a/internal/discover/cache.go b/internal/discover/cache.go
@@ -0,0 +1,80 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import "sync"
+
+type cache struct {
+	d Discover
+
+	sync.Mutex
+	devices []Device
+	hooks   []Hook
+	mounts  []Mount
+}
+
+var _ Discover = (*cache)(nil)
+
+// WithCache decorates the specified disoverer with a cache.
+func WithCache(d Discover) Discover {
+	if d == nil {
+		return None{}
+	}
+	return &cache{d: d}
+}
+
+func (c *cache) Devices() ([]Device, error) {
+	c.Lock()
+	defer c.Unlock()
+
+	if c.devices == nil {
+		devices, err := c.d.Devices()
+		if err != nil {
+			return nil, err
+		}
+		c.devices = devices
+	}
+	return c.devices, nil
+}
+
+func (c *cache) Hooks() ([]Hook, error) {
+	c.Lock()
+	defer c.Unlock()
+
+	if c.hooks == nil {
+		hooks, err := c.d.Hooks()
+		if err != nil {
+			return nil, err
+		}
+		c.hooks = hooks
+	}
+	return c.hooks, nil
+}
+
+func (c *cache) Mounts() ([]Mount, error) {
+	c.Lock()
+	defer c.Unlock()
+
+	if c.mounts == nil {
+		mounts, err := c.d.Mounts()
+		if err != nil {
+			return nil, err
+		}
+		c.mounts = mounts
+	}
+	return c.mounts, nil
+}
diff --git a/internal/discover/first-valid.go b/internal/discover/first-valid.go
@@ -0,0 +1,72 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import "errors"
+
+type firstOf []Discover
+
+// FirstValid returns a discoverer that returns the first non-error result from a list of discoverers.
+func FirstValid(discoverers ...Discover) Discover {
+	var f firstOf
+	for _, d := range discoverers {
+		if d == nil {
+			continue
+		}
+		f = append(f, d)
+	}
+	return f
+}
+
+func (f firstOf) Devices() ([]Device, error) {
+	var errs error
+	for _, d := range f {
+		devices, err := d.Devices()
+		if err != nil {
+			errs = errors.Join(errs, err)
+			continue
+		}
+		return devices, nil
+	}
+	return nil, errs
+}
+
+func (f firstOf) Hooks() ([]Hook, error) {
+	var errs error
+	for _, d := range f {
+		hooks, err := d.Hooks()
+		if err != nil {
+			errs = errors.Join(errs, err)
+			continue
+		}
+		return hooks, nil
+	}
+	return nil, errs
+}
+
+func (f firstOf) Mounts() ([]Mount, error) {
+	var errs error
+	for _, d := range f {
+		mounts, err := d.Mounts()
+		if err != nil {
+			errs = errors.Join(errs, err)
+			continue
+		}
+		return mounts, nil
+	}
+	return nil, nil
+}
diff --git a/internal/platform-support/dgpu/dgpu.go b/internal/platform-support/dgpu/dgpu.go
@@ -17,6 +17,8 @@
 package dgpu
 
 import (
+	"errors"
+
 	"github.com/NVIDIA/go-nvlib/pkg/nvlib/device"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
@@ -25,22 +27,79 @@ import (
 )
 
 // NewForDevice creates a discoverer for the specified Device.
+// nvsandboxutils is used for discovery if specified, otherwise NVML is used.
 func NewForDevice(d device.Device, opts ...Option) (discover.Discover, error) {
 	o := new(opts...)
+	o.isMigDevice = false
+
+	var discoverers []discover.Discover
+	var errs error
+	nvsandboxutilsDiscoverer, err := o.newNvsandboxutilsDGPUDiscoverer(d)
+	if err != nil {
+		// TODO: Log a warning
+		errs = errors.Join(errs, err)
+	} else if nvsandboxutilsDiscoverer != nil {
+		discoverers = append(discoverers, nvsandboxutilsDiscoverer)
+	}
+
+	nvmlDiscoverer, err := o.newNvmlDGPUDiscoverer(&toRequiredInfo{d})
+	if err != nil {
+		// TODO: Log a warning
+		errs = errors.Join(errs, err)
+	} else if nvmlDiscoverer != nil {
+		discoverers = append(discoverers, nvmlDiscoverer)
+	}
+
+	if len(discoverers) == 0 {
+		return nil, errs
+	}
 
-	return o.newNvmlDGPUDiscoverer(&toRequiredInfo{d})
+	return discover.WithCache(
+		discover.FirstValid(
+			discoverers...,
+		),
+	), nil
 }
 
-// NewForDevice creates a discoverer for the specified device and its associated MIG device.
+// NewForMigDevice creates a discoverer for the specified device and its associated MIG device.
+// nvsandboxutils is used for discovery if specified, otherwise NVML is used.
 func NewForMigDevice(d device.Device, mig device.MigDevice, opts ...Option) (discover.Discover, error) {
 	o := new(opts...)
+	o.isMigDevice = true
 
-	return o.newNvmlMigDiscoverer(
+	var discoverers []discover.Discover
+	var errs error
+	nvsandboxutilsDiscoverer, err := o.newNvsandboxutilsDGPUDiscoverer(mig)
+	if err != nil {
+		// TODO: Log a warning
+		errs = errors.Join(errs, err)
+	} else if nvsandboxutilsDiscoverer != nil {
+		discoverers = append(discoverers, nvsandboxutilsDiscoverer)
+	}
+
+	nvmlDiscoverer, err := o.newNvmlMigDiscoverer(
 		&toRequiredMigInfo{
 			MigDevice: mig,
 			parent:    &toRequiredInfo{d},
 		},
 	)
+	if err != nil {
+		// TODO: Log a warning
+		errs = errors.Join(errs, err)
+	} else if nvmlDiscoverer != nil {
+		discoverers = append(discoverers, nvmlDiscoverer)
+	}
+
+	if len(discoverers) == 0 {
+		return nil, errs
+	}
+
+	return discover.WithCache(
+		discover.FirstValid(
+			discoverers...,
+		),
+	), nil
+
 }
 
 func new(opts ...Option) *options {

diff --git a/internal/platform-support/dgpu/nvsandboxutils.go b/internal/platform-support/dgpu/nvsandboxutils.go
@@ -0,0 +1,131 @@
+/**
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package dgpu
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/NVIDIA/go-nvml/pkg/nvml"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/nvsandboxutils"
+)
+
+type nvsandboxutilsDGPU struct {
+	lib               nvsandboxutils.Interface
+	uuid              string
+	devRoot           string
+	isMig             bool
+	nvidiaCDIHookPath string
+	deviceLinks       []string
+}
+
+var _ discover.Discover = (*nvsandboxutilsDGPU)(nil)
+
+type UUIDer interface {
+	GetUUID() (string, nvml.Return)
+}
+
+func (o *options) newNvsandboxutilsDGPUDiscoverer(d UUIDer) (discover.Discover, error) {
+	if o.nvsandboxutilslib == nil {
+		return nil, nil
+	}
+
+	uuid, nvmlRet := d.GetUUID()
+	if nvmlRet != nvml.SUCCESS {
+		return nil, fmt.Errorf("failed to get device UUID: %w", nvmlRet)
+	}
+
+	nvd := nvsandboxutilsDGPU{
+		lib:               o.nvsandboxutilslib,
+		uuid:              uuid,
+		devRoot:           strings.TrimSuffix(filepath.Clean(o.devRoot), "/dev"),
+		isMig:             o.isMigDevice,
+		nvidiaCDIHookPath: o.nvidiaCDIHookPath,
+	}
+
+	return &nvd, nil
+}
+
+func (d *nvsandboxutilsDGPU) Devices() ([]discover.Device, error) {
+	gpuFileInfos, ret := d.lib.GetGpuResource(d.uuid)
+	if ret != nvsandboxutils.SUCCESS {
+		return nil, fmt.Errorf("failed to get GPU resource: %w", ret)
+	}
+
+	var devices []discover.Device
+	for _, info := range gpuFileInfos {
+		switch {
+		case info.SubType == nvsandboxutils.NV_DEV_DRI_CARD, info.SubType == nvsandboxutils.NV_DEV_DRI_RENDERD:
+			if d.isMig {
+				continue
+			}
+			fallthrough
+		case info.SubType == nvsandboxutils.NV_DEV_NVIDIA, info.SubType == nvsandboxutils.NV_DEV_NVIDIA_CAPS_NVIDIA_CAP:
+			containerPath := info.Path
+			if d.devRoot != "/" {
+				containerPath = strings.TrimPrefix(containerPath, d.devRoot)
+			}
+
+			// TODO: Extend discover.Device with additional information.
+			device := discover.Device{
+				HostPath: info.Path,
+				Path:     containerPath,
+			}
+			devices = append(devices, device)
+		case info.SubType == nvsandboxutils.NV_DEV_DRI_CARD_SYMLINK, info.SubType == nvsandboxutils.NV_DEV_DRI_RENDERD_SYMLINK:
+			if d.isMig {
+				continue
+			}
+			if info.Flags == nvsandboxutils.NV_FILE_FLAG_CONTENT {
+				targetPath, ret := d.lib.GetFileContent(info.Path)
+				if ret != nvsandboxutils.SUCCESS {
+					return nil, fmt.Errorf("failed to get symlink: %w", ret)
+				}
+				d.deviceLinks = append(d.deviceLinks, fmt.Sprintf("%v::%v", targetPath, info.Path))
+			}
+		}
+	}
+
+	return devices, nil
+}
+
+// Hooks returns a hook to create the by-path symlinks for the discovered devices.
+func (d *nvsandboxutilsDGPU) Hooks() ([]discover.Hook, error) {
+	if len(d.deviceLinks) == 0 {
+		return nil, nil
+	}
+
+	var args []string
+	for _, l := range d.deviceLinks {
+		args = append(args, "--link", l)
+	}
+
+	hook := discover.CreateNvidiaCDIHook(
+		d.nvidiaCDIHookPath,
+		"create-symlinks",
+		args...,
+	)
+
+	return []discover.Hook{hook}, nil
+}
+
+func (d *nvsandboxutilsDGPU) Mounts() ([]discover.Mount, error) {
+	return nil, nil
+}