feat(import): import single-layer container images directly

Add support to import and overlay single-layer container images. This
allows for a different model of software distribution in the container
image ecosystem.

---

apt-get install openssl curl ca-certificates

(becomes)

    import:
    - path: docker://ghcr.io/homebrew/core/openssl/1.1:1.1.1k
      dest: /
    - path: docker://ghcr.io/homebrew/core/curl:8.0.1
      dest: /
    - path: docker://ghcr.io/homebrew/core/ca-certificates:2022-10-11
      dest: /

Signed-off-by: Ramkumar Chinchani <[email protected]>

Author: rchincha

pkg/oci/oci.go

@@ -2,10 +2,20 @@ package oci
 
 import (
 	"context"
+	"os"
+	"path"
+	"runtime"
+	"sync"
 
+	"github.com/klauspost/pgzip"
+	"github.com/opencontainers/go-digest"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/opencontainers/umoci"
 	"github.com/opencontainers/umoci/oci/casext"
+	"github.com/opencontainers/umoci/oci/layer"
 	"github.com/pkg/errors"
+	"stackerbuild.io/stacker/pkg/log"
+	"stackerbuild.io/stacker/pkg/squashfs"
 )
 
 func LookupManifest(oci casext.Engine, tag string) (ispec.Manifest, error) {
@@ -76,3 +86,108 @@ func UpdateImageConfig(oci casext.Engine, name string, newConfig ispec.Image, ne
 
 	return desc, nil
 }
+
+func hasDirEntries(dir string) bool {
+	ents, err := os.ReadDir(dir)
+	if err != nil {
+		return false
+	}
+	return len(ents) != 0
+}
+
+var tarEx sync.Mutex
+
+// UnpackOne - unpack a single layer (Descriptor) found in ociDir to extractDir
+//
+//	The result of calling unpackOne is either error or the contents available
+//	at the provided extractDir.  The extractDir should be either empty or
+//	fully populated with this layer.
+func UnpackOne(l ispec.Descriptor, ociDir string, extractDir string) error {
+	// population of a dir is not atomic, at least for tar extraction.
+	// As a result, we could hasDirEntries(extractDir) at the same time that
+	// something is un-populating that dir due to a failed extraction (like
+	// os.RemoveAll below).
+	// There needs to be a lock on the extract dir (scoped to the overlay storage backend).
+	// A sync.RWMutex would work well here since it is safe to check as long
+	// as no one is populating or unpopulating.
+	if hasDirEntries(extractDir) {
+		// the directory was already populated.
+		return nil
+	}
+
+	if squashfs.IsSquashfsMediaType(l.MediaType) {
+		return squashfs.ExtractSingleSquash(
+			path.Join(ociDir, "blobs", "sha256", l.Digest.Encoded()), extractDir)
+	}
+	switch l.MediaType {
+	case ispec.MediaTypeImageLayer, ispec.MediaTypeImageLayerGzip:
+		tarEx.Lock()
+		defer tarEx.Unlock()
+
+		oci, err := umoci.OpenLayout(ociDir)
+		if err != nil {
+			return err
+		}
+		defer oci.Close()
+
+		compressed, err := oci.GetBlob(context.Background(), l.Digest)
+		if err != nil {
+			return err
+		}
+		defer compressed.Close()
+
+		uncompressed, err := pgzip.NewReader(compressed)
+		if err != nil {
+			return err
+		}
+
+		err = layer.UnpackLayer(extractDir, uncompressed, nil)
+		if err != nil {
+			if rmErr := os.RemoveAll(extractDir); rmErr != nil {
+				log.Errorf("Failed to remove dir '%s' after failed extraction: %v", extractDir, rmErr)
+			}
+		}
+		return err
+	}
+	return errors.Errorf("unknown media type %s", l.MediaType)
+}
+
+// Unpack an image with "tag" from "ociLayout" into paths returned by "pathfunc"
+func Unpack(ociLayout, tag string, pathfunc func(digest.Digest) string) (int, error) {
+	oci, err := umoci.OpenLayout(ociLayout)
+	if err != nil {
+		return -1, err
+	}
+	defer oci.Close()
+
+	manifest, err := LookupManifest(oci, tag)
+	if err != nil {
+		return -1, err
+	}
+
+	pool := NewThreadPool(runtime.NumCPU())
+
+	seen := map[digest.Digest]bool{}
+	for _, curLayer := range manifest.Layers {
+		// avoid calling UnpackOne twice for the same digest
+		if seen[curLayer.Digest] {
+			continue
+		}
+		seen[curLayer.Digest] = true
+
+		// copy layer to avoid race on pool access.
+		l := curLayer
+		pool.Add(func(ctx context.Context) error {
+			return UnpackOne(l, ociLayout, pathfunc(l.Digest))
+		})
+	}
+
+	pool.DoneAddingJobs()
+
+	err = pool.Run()
+	if err != nil {
+		return -1, err
+	}
+
+	return len(manifest.Layers), nil
+}

pkg/overlay/pool.go renamed to pkg/oci/pool.go

@@ -1,4 +1,4 @@
-package overlay
+package oci
 
 import (
 	"context"

pkg/overlay/overlay.go

@@ -14,6 +14,7 @@ import (
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"golang.org/x/sys/unix"
+	"stackerbuild.io/stacker/pkg/oci"
 	"stackerbuild.io/stacker/pkg/types"
 )
 
@@ -142,14 +143,6 @@ func (o *overlay) SetupEmptyRootfs(name string) error {
 	return ovl.write(o.config, name)
 }
 
-func hasDirEntries(dir string) bool {
-	ents, err := os.ReadDir(dir)
-	if err != nil {
-		return false
-	}
-	return len(ents) != 0
-}
-
 func (o *overlay) snapshot(source string, target string) error {
 	err := o.Create(target)
 	if err != nil {
@@ -168,7 +161,7 @@ func (o *overlay) snapshot(source string, target string) error {
 	}
 	ociDir := path.Join(o.config.StackerDir, "layer-bases", "oci")
 	for _, layer := range manifest.Layers {
-		err := unpackOne(layer, ociDir, overlayPath(o.config.RootFSDir, layer.Digest, "overlay"))
+		err := oci.UnpackOne(layer, ociDir, overlayPath(o.config.RootFSDir, layer.Digest, "overlay"))
 		if err != nil {
 			return errors.Wrapf(err, "Failed mounting %#v", layer)
 		}

pkg/overlay/pack.go

@@ -8,12 +8,9 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"runtime"
 	"strings"
-	"sync"
 	"time"
 
-	"github.com/klauspost/pgzip"
 	"github.com/opencontainers/go-digest"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/opencontainers/umoci"
@@ -24,14 +21,13 @@ import (
 	"github.com/pkg/xattr"
 	"stackerbuild.io/stacker/pkg/lib"
 	"stackerbuild.io/stacker/pkg/log"
+	"stackerbuild.io/stacker/pkg/oci"
 	stackeroci "stackerbuild.io/stacker/pkg/oci"
 	"stackerbuild.io/stacker/pkg/squashfs"
 	"stackerbuild.io/stacker/pkg/storage"
 	"stackerbuild.io/stacker/pkg/types"
 )
 
-var tarEx sync.Mutex
-
 // Container image layers are often tar.gz, however there is nothing in the
 // spec or documentation which standardizes compression params which can cause
 // different layer hashes even for the same tar. So picking compression params
@@ -51,56 +47,6 @@ func overlayPath(rootfs string, d digest.Digest, subdirs ...string) string {
 	return path.Join(dirs...)
 }
 
-func (o *overlay) Unpack(tag, name string) error {
-	cacheDir := path.Join(o.config.StackerDir, "layer-bases", "oci")
-	oci, err := umoci.OpenLayout(cacheDir)
-	if err != nil {
-		return err
-	}
-	defer oci.Close()
-
-	manifest, err := stackeroci.LookupManifest(oci, tag)
-	if err != nil {
-		return err
-	}
-
-	pool := NewThreadPool(runtime.NumCPU())
-
-	seen := map[digest.Digest]bool{}
-	for _, curLayer := range manifest.Layers {
-		// avoid calling unpackOne twice for the same digest
-		if seen[curLayer.Digest] {
-			continue
-		}
-		seen[curLayer.Digest] = true
-
-		// copy layer to avoid race on pool access.
-		l := curLayer
-		pool.Add(func(ctx context.Context) error {
-			return unpackOne(l, cacheDir, overlayPath(o.config.RootFSDir, l.Digest, "overlay"))
-		})
-	}
-
-	pool.DoneAddingJobs()
-
-	err = pool.Run()
-	if err != nil {
-		return err
-	}
-
-	err = o.Create(name)
-	if err != nil {
-		return err
-	}
-
-	ovl, err := newOverlayMetadataFromOCI(oci, tag)
-	if err != nil {
-		return err
-	}
-
-	return ovl.write(o.config, name)
-}
-
 func ConvertAndOutput(config types.StackerConfig, tag, name string, layerType types.LayerType) error {
 	cacheDir := path.Join(config.StackerDir, "layer-bases", "oci")
 	cacheOCI, err := umoci.OpenLayout(cacheDir)
@@ -681,57 +627,32 @@ func repackOverlay(config types.StackerConfig, name string, layerTypes []types.L
 	return ovl.write(config, name)
 }
 
-// unpackOne - unpack a single layer (Descriptor) found in ociDir to extractDir
-//
-//	The result of calling unpackOne is either error or the contents available
-//	at the provided extractDir.  The extractDir should be either empty or
-//	fully populated with this layer.
-func unpackOne(l ispec.Descriptor, ociDir string, extractDir string) error {
-	// population of a dir is not atomic, at least for tar extraction.
-	// As a result, we could hasDirEntries(extractDir) at the same time that
-	// something is un-populating that dir due to a failed extraction (like
-	// os.RemoveAll below).
-	// There needs to be a lock on the extract dir (scoped to the overlay storage backend).
-	// A sync.RWMutex would work well here since it is safe to check as long
-	// as no one is populating or unpopulating.
-	if hasDirEntries(extractDir) {
-		// the directory was already populated.
-		return nil
-	}
+func (o *overlay) Unpack(tag, name string) error {
+	cacheDir := path.Join(o.config.StackerDir, "layer-bases", "oci")
 
-	if squashfs.IsSquashfsMediaType(l.MediaType) {
-		return squashfs.ExtractSingleSquash(
-			path.Join(ociDir, "blobs", "sha256", l.Digest.Encoded()), extractDir)
+	pathfunc := func(digest digest.Digest) string {
+		return overlayPath(o.config.RootFSDir, digest, "overlay")
 	}
-	switch l.MediaType {
-	case ispec.MediaTypeImageLayer, ispec.MediaTypeImageLayerGzip:
-		tarEx.Lock()
-		defer tarEx.Unlock()
 
-		oci, err := umoci.OpenLayout(ociDir)
-		if err != nil {
-			return err
-		}
-		defer oci.Close()
+	_, err := oci.Unpack(cacheDir, tag, pathfunc)
+	if err != nil {
+		return err
+	}
 
-		compressed, err := oci.GetBlob(context.Background(), l.Digest)
-		if err != nil {
-			return err
-		}
-		defer compressed.Close()
+	err = o.Create(name)
+	if err != nil {
+		return err
+	}
 
-		uncompressed, err := pgzip.NewReader(compressed)
-		if err != nil {
-			return err
-		}
+	oci, err := umoci.OpenLayout(cacheDir)
+	if err != nil {
+		return err
+	}
 
-		err = layer.UnpackLayer(extractDir, uncompressed, nil)
-		if err != nil {
-			if rmErr := os.RemoveAll(extractDir); rmErr != nil {
-				log.Errorf("Failed to remove dir '%s' after failed extraction: %v", extractDir, rmErr)
-			}
-		}
+	ovl, err := newOverlayMetadataFromOCI(oci, tag)
+	if err != nil {
 		return err
 	}
-	return errors.Errorf("unknown media type %s", l.MediaType)
+
+	return ovl.write(o.config, name)
 }

pkg/stacker/import.go

@@ -13,6 +13,7 @@ import (
 	"github.com/vbatts/go-mtree"
 	"stackerbuild.io/stacker/pkg/lib"
 	"stackerbuild.io/stacker/pkg/log"
+	"stackerbuild.io/stacker/pkg/oci"
 	"stackerbuild.io/stacker/pkg/types"
 )
 
@@ -294,6 +295,38 @@ func acquireUrl(c types.StackerConfig, storage types.Storage, i string, cache st
 		}
 
 		return p, nil
+	} else if url.Scheme == "docker" {
+		if idest != "" && idest[len(idest)-1:] != "/" {
+			return "", errors.Errorf("The destination path must be directory: %s", idest)
+		}
+
+		is := types.ImageSource{Type: "docker", Url: i}
+		if err := importContainersImage(is, c, false); err != nil {
+			return "", err
+		}
+
+		tag, err := is.ParseTag()
+		if err != nil {
+			return "", err
+		}
+
+		pathfunc := func(digest digest.Digest) string {
+			_ = os.Remove(cache)
+			return cache
+		}
+
+		ociDir := path.Join(c.StackerDir, "layer-bases", "oci")
+
+		n, err := oci.Unpack(ociDir, tag, pathfunc)
+		if err != nil {
+			return "", err
+		}
+
+		if n > 1 {
+			return "", errors.Errorf("Currently supporting single-layer container image imports")
+		}
+
+		return cache, nil
 	}
 
 	return "", errors.Errorf("unsupported url scheme %s", i)