diff --git a/go.mod b/go.mod index b2903d51b5..80f30ec663 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/containers/libhvee v0.7.1 github.com/containers/ocicrypt v1.2.0 github.com/containers/psgo v1.9.0 - github.com/containers/storage v1.55.1-0.20240821103551-8ec73cadc730 + github.com/containers/storage v1.55.1-0.20240903205438-465c38f89483 github.com/containers/winquit v1.1.0 github.com/coreos/go-systemd/v22 v22.5.1-0.20231103132048-7d375ecc2b09 github.com/coreos/stream-metadata-go v0.4.4 diff --git a/go.sum b/go.sum index b0586f4053..1790555527 100644 --- a/go.sum +++ b/go.sum @@ -99,8 +99,8 @@ github.com/containers/ocicrypt v1.2.0 h1:X14EgRK3xNFvJEfI5O4Qn4T3E25ANudSOZz/sir github.com/containers/ocicrypt v1.2.0/go.mod h1:ZNviigQajtdlxIZGibvblVuIFBKIuUI2M0QM12SD31U= github.com/containers/psgo v1.9.0 h1:eJ74jzSaCHnWt26OlKZROSyUyRcGDf+gYBdXnxrMW4g= github.com/containers/psgo v1.9.0/go.mod h1:0YoluUm43Mz2UnBIh1P+6V6NWcbpTL5uRtXyOcH0B5A= -github.com/containers/storage v1.55.1-0.20240821103551-8ec73cadc730 h1:rFhDkjeR52VGko+lUd7veRbfdfMhnNBL0cniGudunCM= -github.com/containers/storage v1.55.1-0.20240821103551-8ec73cadc730/go.mod h1:oDe+z/9gI/Fa4NKfTTaPGVAaRbDJnHWwtR3yntqYz8M= +github.com/containers/storage v1.55.1-0.20240903205438-465c38f89483 h1:hQOAlIad+xjukeGFHQbH/x5I2zuPNCXmjvSrxX5ERF4= +github.com/containers/storage v1.55.1-0.20240903205438-465c38f89483/go.mod h1:fRTU33KP5BXpOIWDxDgU5LpHbrOzWxmVmtm/3PYLlgE= github.com/containers/winquit v1.1.0 h1:jArun04BNDQvt2W0Y78kh9TazN2EIEMG5Im6/JY7+pE= github.com/containers/winquit v1.1.0/go.mod h1:PsPeZlnbkmGGIToMPHF1zhWjBUkd8aHjMOr/vFcPxw8= github.com/coreos/go-oidc/v3 v3.10.0 h1:tDnXHnLyiTVyT/2zLDGj09pFPkhND8Gl8lnTRhoEaJU= diff --git a/vendor/github.com/containers/storage/.cirrus.yml b/vendor/github.com/containers/storage/.cirrus.yml index 50b9876169..887147040d 100644 --- a/vendor/github.com/containers/storage/.cirrus.yml +++ b/vendor/github.com/containers/storage/.cirrus.yml @@ -23,7 +23,7 @@ env: # GCE project where images live IMAGE_PROJECT: "libpod-218412" # VM Image built in containers/automation_images - IMAGE_SUFFIX: "c20240529t141726z-f40f39d13" + IMAGE_SUFFIX: "c20240821t171500z-f40f39d13" FEDORA_CACHE_IMAGE_NAME: "fedora-${IMAGE_SUFFIX}" DEBIAN_CACHE_IMAGE_NAME: "debian-${IMAGE_SUFFIX}" diff --git a/vendor/github.com/containers/storage/Makefile b/vendor/github.com/containers/storage/Makefile index 8e7d79bc24..6f20e059d5 100644 --- a/vendor/github.com/containers/storage/Makefile +++ b/vendor/github.com/containers/storage/Makefile @@ -35,7 +35,7 @@ TESTFLAGS := $(shell $(GO) test -race $(BUILDFLAGS) ./pkg/stringutils 2>&1 > /de # N/B: This value is managed by Renovate, manual changes are # possible, as long as they don't disturb the formatting # (i.e. DO NOT ADD A 'v' prefix!) -GOLANGCI_LINT_VERSION := 1.60.2 +GOLANGCI_LINT_VERSION := 1.60.3 default all: local-binary docs local-validate local-cross ## validate all checks, build and cross-build\nbinaries and docs diff --git a/vendor/github.com/containers/storage/drivers/overlay/composefs.go b/vendor/github.com/containers/storage/drivers/overlay/composefs.go index 6f36ba3dda..e1fe7bb554 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/composefs.go +++ b/vendor/github.com/containers/storage/drivers/overlay/composefs.go @@ -137,54 +137,62 @@ func hasACL(path string) (bool, error) { return binary.LittleEndian.Uint32(flags)&LCFS_EROFS_FLAGS_HAS_ACL != 0, nil } -func mountComposefsBlob(dataDir, mountPoint string) error { +func openComposefsMount(dataDir string) (int, error) { blobFile := getComposefsBlob(dataDir) loop, err := loopback.AttachLoopDeviceRO(blobFile) if err != nil { - return err + return -1, err } defer loop.Close() hasACL, err := hasACL(blobFile) if err != nil { - return err + return -1, err } fsfd, err := unix.Fsopen("erofs", 0) if err != nil { - return fmt.Errorf("failed to open erofs filesystem: %w", err) + return -1, fmt.Errorf("failed to open erofs filesystem: %w", err) } defer unix.Close(fsfd) if err := unix.FsconfigSetString(fsfd, "source", loop.Name()); err != nil { - return fmt.Errorf("failed to set source for erofs filesystem: %w", err) + return -1, fmt.Errorf("failed to set source for erofs filesystem: %w", err) } if err := unix.FsconfigSetFlag(fsfd, "ro"); err != nil { - return fmt.Errorf("failed to set erofs filesystem read-only: %w", err) + return -1, fmt.Errorf("failed to set erofs filesystem read-only: %w", err) } if !hasACL { if err := unix.FsconfigSetFlag(fsfd, "noacl"); err != nil { - return fmt.Errorf("failed to set noacl for erofs filesystem: %w", err) + return -1, fmt.Errorf("failed to set noacl for erofs filesystem: %w", err) } } if err := unix.FsconfigCreate(fsfd); err != nil { buffer := make([]byte, 4096) if n, _ := unix.Read(fsfd, buffer); n > 0 { - return fmt.Errorf("failed to create erofs filesystem: %s: %w", string(buffer[:n]), err) + return -1, fmt.Errorf("failed to create erofs filesystem: %s: %w", string(buffer[:n]), err) } - return fmt.Errorf("failed to create erofs filesystem: %w", err) + return -1, fmt.Errorf("failed to create erofs filesystem: %w", err) } mfd, err := unix.Fsmount(fsfd, 0, unix.MOUNT_ATTR_RDONLY) if err != nil { buffer := make([]byte, 4096) if n, _ := unix.Read(fsfd, buffer); n > 0 { - return fmt.Errorf("failed to mount erofs filesystem: %s: %w", string(buffer[:n]), err) + return -1, fmt.Errorf("failed to mount erofs filesystem: %s: %w", string(buffer[:n]), err) } - return fmt.Errorf("failed to mount erofs filesystem: %w", err) + return -1, fmt.Errorf("failed to mount erofs filesystem: %w", err) + } + return mfd, nil +} + +func mountComposefsBlob(dataDir, mountPoint string) error { + mfd, err := openComposefsMount(dataDir) + if err != nil { + return err } defer unix.Close(mfd) diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay.go b/vendor/github.com/containers/storage/drivers/overlay/overlay.go index 9b24336f0b..63777fe470 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay.go @@ -1456,6 +1456,35 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO return "", err } + // user namespace requires this to move a directory from lower to upper. + rootUID, rootGID, err := idtools.GetRootUIDGID(options.UidMaps, options.GidMaps) + if err != nil { + return "", err + } + + mergedDir := d.getMergedDir(id, dir, inAdditionalStore) + // Attempt to create the merged dir if it doesn't exist, but don't chown an already existing directory (it might be in an additional store) + if err := idtools.MkdirAllAndChownNew(mergedDir, 0o700, idtools.IDPair{UID: rootUID, GID: rootGID}); err != nil && !os.IsExist(err) { + return "", err + } + + if count := d.ctr.Increment(mergedDir); count > 1 { + return mergedDir, nil + } + defer func() { + if retErr != nil { + if c := d.ctr.Decrement(mergedDir); c <= 0 { + if mntErr := unix.Unmount(mergedDir, 0); mntErr != nil { + // Ignore EINVAL, it means the directory is not a mount point and it can happen + // if the current function fails before the mount point is created. + if !errors.Is(mntErr, unix.EINVAL) { + logrus.Errorf("Unmounting %v: %v", mergedDir, mntErr) + } + } + } + } + }() + readWrite := !inAdditionalStore if !d.SupportsShifting() || options.DisableShifting { @@ -1575,7 +1604,7 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO return "", fmt.Errorf("cannot mount a composefs layer as writeable") } - dest := filepath.Join(composeFsLayersDir, fmt.Sprintf("%d", i)) + dest := filepath.Join(composeFsLayersDir, strconv.Itoa(i)) if err := os.MkdirAll(dest, 0o700); err != nil { return "", err } @@ -1683,12 +1712,6 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO optsList = append(optsList, "metacopy=on", "redirect_dir=on") } - // user namespace requires this to move a directory from lower to upper. - rootUID, rootGID, err := idtools.GetRootUIDGID(options.UidMaps, options.GidMaps) - if err != nil { - return "", err - } - if len(absLowers) == 0 { absLowers = append(absLowers, path.Join(dir, "empty")) } @@ -1703,26 +1726,6 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO } } - mergedDir := d.getMergedDir(id, dir, inAdditionalStore) - // Attempt to create the merged dir only if it doesn't exist. - if err := fileutils.Exists(mergedDir); err != nil && os.IsNotExist(err) { - if err := idtools.MkdirAllAs(mergedDir, 0o700, rootUID, rootGID); err != nil && !os.IsExist(err) { - return "", err - } - } - if count := d.ctr.Increment(mergedDir); count > 1 { - return mergedDir, nil - } - defer func() { - if retErr != nil { - if c := d.ctr.Decrement(mergedDir); c <= 0 { - if mntErr := unix.Unmount(mergedDir, 0); mntErr != nil { - logrus.Errorf("Unmounting %v: %v", mergedDir, mntErr) - } - } - } - }() - workdir := path.Join(dir, "work") if d.options.mountProgram == "" && unshare.IsRootless() { @@ -1879,10 +1882,21 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO // getMergedDir returns the directory path that should be used as the mount point for the overlayfs. func (d *Driver) getMergedDir(id, dir string, inAdditionalStore bool) string { - // If the layer is in an additional store, the lock we might hold only a reading lock. To prevent - // races with other processes, use a private directory under the main store rundir. At this point, the - // current process is holding an exclusive lock on the store, and since the rundir cannot be shared for - // different stores, it is safe to assume the current process has exclusive access to it. + // Ordinarily, .Get() (layer mounting) callers are supposed to guarantee exclusion. + // + // But additional stores are initialized with RO locks and don’t support a write + // lock operation at all; and naiveDiff operations cause mounts/unmounts, so they might + // happen on code paths where we might only holding a RO lock for the additional store. + // To prevent races with other processes mounting or unmounting the layer, + // use a private directory under the main store rundir, not the "merged" directory inside the + // original layer store holding the layer data. + // + // To support this, contrary to the _general_ locking rules for .Diff / .Changes (which allow a RO lock), + // the top-level Store implementation uses an exclusive lock for the primary layer store; + // and since the rundir cannot be shared for different stores, it is safe to assume the + // current process has exclusive access to it. + // + // LOCKING BUG? the .DiffSize operation does not currently hold an exclusive lock on the primary store. if inAdditionalStore { return path.Join(d.runhome, id, "merged") } @@ -2128,24 +2142,16 @@ func (d *Driver) DiffGetter(id string) (_ graphdriver.FileGetCloser, Err error) for _, diffDir := range diffDirs { // diffDir has the form $GRAPH_ROOT/overlay/$ID/diff, so grab the $ID from the parent directory id := path.Base(path.Dir(diffDir)) - composefsBlob := d.getComposefsData(id) - if fileutils.Exists(composefsBlob) != nil { + composefsData := d.getComposefsData(id) + if fileutils.Exists(composefsData) != nil { // not a composefs layer, ignore it continue } - dir, err := os.MkdirTemp(d.runhome, "composefs-mnt") - if err != nil { - return nil, err - } - if err := mountComposefsBlob(composefsBlob, dir); err != nil { - return nil, err - } - fd, err := os.Open(dir) + fd, err := openComposefsMount(composefsData) if err != nil { return nil, err } - composefsMounts[diffDir] = fd - _ = unix.Unmount(dir, unix.MNT_DETACH) + composefsMounts[diffDir] = os.NewFile(uintptr(fd), composefsData) } return &overlayFileGetter{diffDirs: diffDirs, composefsMounts: composefsMounts}, nil } diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay_nocgo.go b/vendor/github.com/containers/storage/drivers/overlay/overlay_nocgo.go index 0577711b3d..313f1d6a31 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay_nocgo.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay_nocgo.go @@ -7,6 +7,10 @@ import ( "fmt" ) +func openComposefsMount(dataDir string) (int, error) { + return 0, fmt.Errorf("composefs not supported on this build") +} + func getComposeFsHelper() (string, error) { return "", fmt.Errorf("composefs not supported on this build") } diff --git a/vendor/github.com/containers/storage/layers.go b/vendor/github.com/containers/storage/layers.go index 6caf28ab71..8ae969894d 100644 --- a/vendor/github.com/containers/storage/layers.go +++ b/vendor/github.com/containers/storage/layers.go @@ -2095,6 +2095,9 @@ func (r *layerStore) layerMappings(layer *Layer) *idtools.IDMappings { } // Requires startReading or startWriting. +// +// NOTE: Overlay’s implementation assumes use of an exclusive lock over the primary layer store, +// see drivers/overlay.Driver.getMergedDir. func (r *layerStore) Changes(from, to string) ([]archive.Change, error) { from, to, fromLayer, toLayer, err := r.findParentAndLayer(from, to) if err != nil { @@ -2161,6 +2164,9 @@ func writeCompressedDataGoroutine(pwriter *io.PipeWriter, compressor io.WriteClo } // Requires startReading or startWriting. +// +// NOTE: Overlay’s implementation assumes use of an exclusive lock over the primary layer store, +// see drivers/overlay.Driver.getMergedDir. func (r *layerStore) Diff(from, to string, options *DiffOptions) (io.ReadCloser, error) { var metadata storage.Unpacker diff --git a/vendor/github.com/containers/storage/pkg/chunked/cache_linux.go b/vendor/github.com/containers/storage/pkg/chunked/cache_linux.go index d49ddfed03..1d823c8d49 100644 --- a/vendor/github.com/containers/storage/pkg/chunked/cache_linux.go +++ b/vendor/github.com/containers/storage/pkg/chunked/cache_linux.go @@ -289,8 +289,10 @@ func (c *layersCache) load() error { } if r.ReadOnly { - // if the layer is coming from a read-only store, do not attempt + // If the layer is coming from a read-only store, do not attempt // to write to it. + // Therefore,we won’t find any matches in read-only-store layers, + // unless the read-only store layer comes prepopulated with cacheKey data. continue } diff --git a/vendor/github.com/containers/storage/pkg/fileutils/exists_freebsd.go b/vendor/github.com/containers/storage/pkg/fileutils/exists_freebsd.go new file mode 100644 index 0000000000..eeecc9f75e --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/fileutils/exists_freebsd.go @@ -0,0 +1,38 @@ +package fileutils + +import ( + "errors" + "os" + "syscall" + + "golang.org/x/sys/unix" +) + +// Exists checks whether a file or directory exists at the given path. +// If the path is a symlink, the symlink is followed. +func Exists(path string) error { + // It uses unix.Faccessat which is a faster operation compared to os.Stat for + // simply checking the existence of a file. + err := unix.Faccessat(unix.AT_FDCWD, path, unix.F_OK, 0) + if err != nil { + return &os.PathError{Op: "faccessat", Path: path, Err: err} + } + return nil +} + +// Lexists checks whether a file or directory exists at the given path. +// If the path is a symlink, the symlink itself is checked. +func Lexists(path string) error { + // FreeBSD before 15.0 does not support the AT_SYMLINK_NOFOLLOW flag for + // faccessat. In this case, the call to faccessat will return EINVAL and + // we fall back to using Lstat. + err := unix.Faccessat(unix.AT_FDCWD, path, unix.F_OK, unix.AT_SYMLINK_NOFOLLOW) + if err != nil { + if errors.Is(err, syscall.EINVAL) { + _, err = os.Lstat(path) + return err + } + return &os.PathError{Op: "faccessat", Path: path, Err: err} + } + return nil +} diff --git a/vendor/github.com/containers/storage/pkg/fileutils/exists_unix.go b/vendor/github.com/containers/storage/pkg/fileutils/exists_unix.go index f3087d7df6..56d27d869d 100644 --- a/vendor/github.com/containers/storage/pkg/fileutils/exists_unix.go +++ b/vendor/github.com/containers/storage/pkg/fileutils/exists_unix.go @@ -1,5 +1,5 @@ -//go:build !windows -// +build !windows +//go:build !windows && !freebsd +// +build !windows,!freebsd package fileutils diff --git a/vendor/github.com/containers/storage/storage.conf b/vendor/github.com/containers/storage/storage.conf index 779bf16b8c..7ac8fdf8f0 100644 --- a/vendor/github.com/containers/storage/storage.conf +++ b/vendor/github.com/containers/storage/storage.conf @@ -70,13 +70,13 @@ additionalimagestores = [ # Path to an ostree repository that might have # previously pulled content which can be used when attempting to avoid -# pulling content from the container registry +# pulling content from the container registry. # ostree_repos="" -# If set to "true", containers/storage will convert images to a -# format compatible with partial pulls in order to take advantage -# of local deduplication and hard linking. It is an expensive -# operation so it is not enabled by default. +# If set to "true", containers/storage will convert images that are +# not already in zstd:chunked format to that format before processing +# in order to take advantage of local deduplication and hard linking. +# It is an expensive operation so it is not enabled by default. # This is a "string bool": "false" | "true" (cannot be native TOML boolean) # convert_images = "false" diff --git a/vendor/github.com/containers/storage/store.go b/vendor/github.com/containers/storage/store.go index 8510d033c6..bd4da7a468 100644 --- a/vendor/github.com/containers/storage/store.go +++ b/vendor/github.com/containers/storage/store.go @@ -2963,6 +2963,10 @@ func (s *store) Changes(from, to string) ([]archive.Change, error) { if err != nil { return nil, err } + + // While the general rules require the layer store to only be locked RO (apart from known LOCKING BUGs) + // the overlay driver requires the primary layer store to be locked RW; see + // drivers/overlay.Driver.getMergedDir. if err := rlstore.startWriting(); err != nil { return nil, err } @@ -3019,6 +3023,9 @@ func (s *store) Diff(from, to string, options *DiffOptions) (io.ReadCloser, erro return nil, err } + // While the general rules require the layer store to only be locked RO (apart from known LOCKING BUGs) + // the overlay driver requires the primary layer store to be locked RW; see + // drivers/overlay.Driver.getMergedDir. if err := rlstore.startWriting(); err != nil { return nil, err } diff --git a/vendor/modules.txt b/vendor/modules.txt index 905d73c6e1..2c7e64ee04 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -355,7 +355,7 @@ github.com/containers/psgo/internal/dev github.com/containers/psgo/internal/host github.com/containers/psgo/internal/proc github.com/containers/psgo/internal/process -# github.com/containers/storage v1.55.1-0.20240821103551-8ec73cadc730 +# github.com/containers/storage v1.55.1-0.20240903205438-465c38f89483 ## explicit; go 1.21 github.com/containers/storage github.com/containers/storage/drivers