fix(remount): relocate libraries along with their symlinks

envbuilder.go

@@ -438,7 +438,7 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath)
 	tempRemountDest := filepath.Join("/", MagicDir, "mnt")
 	// ignorePrefixes is a superset of ignorePaths that we pass to kaniko's
 	// IgnoreList.
-	ignorePrefixes := append([]string{"/proc", "/sys"}, ignorePaths...)
+	ignorePrefixes := append([]string{"/dev", "/proc", "/sys"}, ignorePaths...)
 	restoreMounts, err := ebutil.TempRemount(options.Logger, tempRemountDest, ignorePrefixes...)
 	defer func() { // restoreMounts should never be nil
 		if err := restoreMounts(); err != nil {

internal/ebutil/libs.go

@@ -0,0 +1,86 @@
+package ebutil
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+// Container runtimes like NVIDIA mount individual libraries into the container
+// (e.g. `<libname>.so.<driver_version>`) and create symlinks for them
+// (e.g. `<libname>.so.1`). This code helps with finding the right library
+// directory for the target Linux distribution as well as locating the symlinks.
+//
+// Please see [#143 (comment)] for further details.
+//
+// [#143 (comment)]: https://github.com/coder/envbuilder/issues/143#issuecomment-2192405828
+
+// Based on https://github.com/NVIDIA/libnvidia-container/blob/v1.15.0/src/common.h#L29
+const usrLibDir = "/usr/lib64"
+
+const debianVersionFile = "/etc/debian_version"
+
+// libraryDirectoryPath returns the library directory. It returns a multiarch
+// directory if the distribution is Debian or a derivative.
+//
+// Based on https://github.com/NVIDIA/libnvidia-container/blob/v1.15.0/src/nvc_container.c#L152-L165
+func libraryDirectoryPath(m mounter) (string, error) {
+	// Debian and its derivatives use a multiarch directory scheme.
+	if _, err := m.Stat(debianVersionFile); err != nil && !errors.Is(err, os.ErrNotExist) {
+		return "", fmt.Errorf("check if debian: %w", err)
+	} else if err == nil {
+		return usrLibMultiarchDir, nil
+	}
+
+	return usrLibDir, nil
+}
+
+// libraryDirectorySymlinks returns a mapping of each library (basename) with a
+// list of their symlinks (basename). Libraries with no symlinks do not appear
+// in the mapping.
+func libraryDirectorySymlinks(m mounter, libDir string) (map[string][]string, error) {
+	des, err := m.ReadDir(libDir)
+	if err != nil {
+		return nil, fmt.Errorf("read directory %s: %w", libDir, err)
+	}
+
+	libsSymlinks := make(map[string][]string)
+	for _, de := range des {
+		if de.IsDir() {
+			continue
+		}
+
+		if de.Type()&os.ModeSymlink != os.ModeSymlink {
+			// Not a symlink. Skip.
+			continue
+		}
+
+		symlink := filepath.Join(libDir, de.Name())
+		path, err := m.EvalSymlinks(symlink)
+		if err != nil {
+			return nil, fmt.Errorf("eval symlink %s: %w", symlink, err)
+		}
+
+		path = filepath.Base(path)
+		if _, ok := libsSymlinks[path]; !ok {
+			libsSymlinks[path] = make([]string, 0, 1)
+		}
+
+		libsSymlinks[path] = append(libsSymlinks[path], de.Name())
+	}
+
+	return libsSymlinks, nil
+}
+
+// moveLibSymlinks moves a list of symlinks from source to destination directory.
+func moveLibSymlinks(m mounter, symlinks []string, srcDir, destDir string) error {
+	for _, l := range symlinks {
+		oldpath := filepath.Join(srcDir, l)
+		newpath := filepath.Join(destDir, l)
+		if err := m.Rename(oldpath, newpath); err != nil {
+			return fmt.Errorf("move symlink %s => %s: %w", oldpath, newpath, err)
+		}
+	}
+	return nil
+}

internal/ebutil/libs_amd64.go

@@ -0,0 +1,7 @@
+//go:build amd64
+
+package ebutil
+
+// Based on https://github.com/NVIDIA/libnvidia-container/blob/v1.15.0/src/common.h#L36
+
+const usrLibMultiarchDir = "/usr/lib/x86_64-linux-gnu"

internal/ebutil/libs_arm64.go

@@ -0,0 +1,7 @@
+//go:build arm64
+
+package ebutil
+
+// Based on https://github.com/NVIDIA/libnvidia-container/blob/v1.15.0/src/common.h#L52
+
+const usrLibMultiarchDir = "/usr/lib/aarch64-linux-gnu"

internal/ebutil/remount.go

@@ -1,6 +1,7 @@
 package ebutil
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -44,15 +45,36 @@ func tempRemount(m mounter, logf func(notcodersdk.LogLevel, string, ...any), bas
 		return func() error { return nil }, fmt.Errorf("get mounts: %w", err)
 	}
 
+	libDir, err := libraryDirectoryPath(m)
+	if err != nil {
+		return func() error { return nil }, fmt.Errorf("get lib directory: %w", err)
+	}
+
+	libsSymlinks, err := libraryDirectorySymlinks(m, libDir)
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return func() error { return nil }, fmt.Errorf("read lib symlinks: %w", err)
+	}
+
 	// temp move of all ro mounts
 	mounts := map[string]string{}
 	var restoreOnce sync.Once
 	var merr error
 	// closer to attempt to restore original mount points
 	restore = func() error {
 		restoreOnce.Do(func() {
+			if len(mounts) == 0 {
+				return
+			}
+
+			newLibDir, err := libraryDirectoryPath(m)
+			if err != nil {
+				merr = multierror.Append(merr, fmt.Errorf("get new lib directory: %w", err))
+				return
+			}
+
 			for orig, moved := range mounts {
-				if err := remount(m, moved, orig); err != nil {
+				logf(notcodersdk.LogLevelTrace, "restore mount %s", orig)
+				if err := remount(m, moved, orig, newLibDir, libsSymlinks); err != nil {
 					merr = multierror.Append(merr, fmt.Errorf("restore mount: %w", err))
 				}
 			}
@@ -77,7 +99,8 @@ outer:
 
 		src := mountInfo.MountPoint
 		dest := filepath.Join(base, src)
-		if err := remount(m, src, dest); err != nil {
+		logf(notcodersdk.LogLevelTrace, "temp remount %s", src)
+		if err := remount(m, src, dest, libDir, libsSymlinks); err != nil {
 			return restore, fmt.Errorf("temp remount: %w", err)
 		}
 
@@ -87,30 +110,48 @@ outer:
 	return restore, nil
 }
 
-func remount(m mounter, src, dest string) error {
+func remount(m mounter, src, dest, libDir string, libsSymlinks map[string][]string) error {
 	stat, err := m.Stat(src)
 	if err != nil {
 		return fmt.Errorf("stat %s: %w", src, err)
 	}
+
 	var destDir string
 	if stat.IsDir() {
 		destDir = dest
 	} else {
 		destDir = filepath.Dir(dest)
+		if destDir == usrLibDir || destDir == usrLibMultiarchDir {
+			// Restore mount to libDir
+			destDir = libDir
+			dest = filepath.Join(destDir, stat.Name())
+		}
 	}
+
 	if err := m.MkdirAll(destDir, 0o750); err != nil {
 		return fmt.Errorf("ensure path: %w", err)
 	}
+
 	if !stat.IsDir() {
 		f, err := m.OpenFile(dest, os.O_CREATE, 0o640)
 		if err != nil {
 			return fmt.Errorf("ensure file path: %w", err)
 		}
-		defer f.Close()
+		// This ensure the file is created, it will not be used. It can be closed immediately.
+		f.Close()
+
+		if symlinks, ok := libsSymlinks[stat.Name()]; ok {
+			srcDir := filepath.Dir(src)
+			if err := moveLibSymlinks(m, symlinks, srcDir, destDir); err != nil {
+				return err
+			}
+		}
 	}
+
 	if err := m.Mount(src, dest, "bind", syscall.MS_BIND, ""); err != nil {
 		return fmt.Errorf("bind mount %s => %s: %w", src, dest, err)
 	}
+
 	if err := m.Unmount(src, 0); err != nil {
 		return fmt.Errorf("unmount orig src %s: %w", src, err)
 	}
@@ -131,6 +172,12 @@ type mounter interface {
 	Mount(string, string, string, uintptr, string) error
 	// Unmount wraps syscall.Unmount
 	Unmount(string, int) error
+	// ReadDir wraps os.ReadDir
+	ReadDir(string) ([]os.DirEntry, error)
+	// EvalSymlinks wraps filepath.EvalSymlinks
+	EvalSymlinks(string) (string, error)
+	// Rename wraps os.Rename
+	Rename(string, string) error
 }
 
 // realMounter implements mounter and actually does the thing.
@@ -161,3 +208,15 @@ func (m *realMounter) OpenFile(name string, flag int, perm os.FileMode) (*os.Fil
 func (m *realMounter) Stat(path string) (os.FileInfo, error) {
 	return os.Stat(path)
 }
+
+func (m *realMounter) ReadDir(name string) ([]os.DirEntry, error) {
+	return os.ReadDir(name)
+}
+
+func (m *realMounter) EvalSymlinks(path string) (string, error) {
+	return filepath.EvalSymlinks(path)
+}
+
+func (m *realMounter) Rename(oldpath, newpath string) error {
+	return os.Rename(oldpath, newpath)
+}