From d064762af54607b2d2ce63b58ab693a7707e8af7 Mon Sep 17 00:00:00 2001 From: Mike P Date: Fri, 30 Aug 2024 08:36:02 +0100 Subject: [PATCH 1/2] Add hashbang to top-level test script This fixes the "Exec format error" when running the podman tests. --- tests/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/default.nix b/tests/default.nix index a0b0b7f..755e8c8 100644 --- a/tests/default.nix +++ b/tests/default.nix @@ -173,6 +173,7 @@ let "\n" (pkgs.lib.mapAttrsToList (n: v: "echo Running test '${n}'\n${v}/bin/test-script") tests); in pkgs.writeScriptBin "all-test-scripts" '' + #!${pkgs.runtimeShell} set -e ${scripts} ''; From 92b5dddb131c64cfc1e21df1c6c246299198d60d Mon Sep 17 00:00:00 2001 From: Mike P Date: Thu, 22 Aug 2024 11:32:18 +0100 Subject: [PATCH 2/2] Allow layer author, comment, and created-by metadata to be set --- README.md | 2 ++ cmd/layers.go | 27 +++++++++++++++++++++++++-- cmd/utils.go | 14 ++++++++++++++ default.nix | 5 +++++ examples/default.nix | 1 + examples/metadata.nix | 17 +++++++++++++++++ nix/image.go | 13 ++++++------- nix/image_test.go | 3 +++ nix/layers.go | 11 ++++++----- nix/layers_test.go | 8 +++++--- tests/default.nix | 40 +++++++++++++++++++++++++++++++++++++++- types/types.go | 1 + 12 files changed, 124 insertions(+), 18 deletions(-) create mode 100644 examples/metadata.nix diff --git a/README.md b/README.md index 17084ee..6d299c9 100644 --- a/README.md +++ b/README.md @@ -269,6 +269,8 @@ Function arguments are: building the layer. This is mainly useful to ignore the configuration file from the container layer. +- **`metadata`** (defaults to `{ created_by = "nix2container"; }`): an attribute + set containing this layer's `created_by`, `author` and `comment` values ## Isolate dependencies in dedicated layers diff --git a/cmd/layers.go b/cmd/layers.go index 90918ef..f204fc6 100644 --- a/cmd/layers.go +++ b/cmd/layers.go @@ -17,12 +17,15 @@ import ( "github.com/nlewo/nix2container/types" "github.com/sirupsen/logrus" "github.com/spf13/cobra" + + v1 "github.com/opencontainers/image-spec/specs-go/v1" ) var ignore string var tarDirectory string var permsFilepath string var rewritesFilepath string +var historyFilepath string var maxLayers int // layerCmd represents the layer command @@ -62,7 +65,16 @@ var layersReproducibleCmd = &cobra.Command{ os.Exit(1) } } - layers, err := nix.NewLayers(storepaths, maxLayers, parents, rewrites, ignore, perms) + var history v1.History + if historyFilepath != "" { + history, err = readHistoryFile(historyFilepath) + if err != nil { + fmt.Fprintf(os.Stderr, "%s", err) + os.Exit(1) + } + } + + layers, err := nix.NewLayers(storepaths, maxLayers, parents, rewrites, ignore, perms, history) if err != nil { fmt.Fprintf(os.Stderr, "%s", err) os.Exit(1) @@ -112,7 +124,16 @@ var layersNonReproducibleCmd = &cobra.Command{ os.Exit(1) } } - layers, err := nix.NewLayersNonReproducible(storepaths, maxLayers, tarDirectory, parents, rewrites, ignore, perms) + var history v1.History + if historyFilepath != "" { + history, err = readHistoryFile(historyFilepath) + if err != nil { + fmt.Fprintf(os.Stderr, "%s", err) + os.Exit(1) + } + } + + layers, err := nix.NewLayersNonReproducible(storepaths, maxLayers, tarDirectory, parents, rewrites, ignore, perms, history) if err != nil { fmt.Fprintf(os.Stderr, "%s", err) os.Exit(1) @@ -157,12 +178,14 @@ func init() { layersNonReproducibleCmd.Flags().StringVarP(&rewritesFilepath, "rewrites", "", "", "A JSON file containing a list of path rewrites. Each element of the list is a JSON object with the attributes path, regex and repl: for a given path, the regex is replaced by repl.") layersNonReproducibleCmd.Flags().StringVarP(&permsFilepath, "perms", "", "", "A JSON file containing file permissions") + layersNonReproducibleCmd.Flags().StringVarP(&historyFilepath, "history", "", "", "A JSON file containing layer history") layersNonReproducibleCmd.Flags().IntVarP(&maxLayers, "max-layers", "", 1, "The maximum number of layers") rootCmd.AddCommand(layersReproducibleCmd) layersReproducibleCmd.Flags().StringVarP(&ignore, "ignore", "", "", "Ignore the path from the list of storepaths") layersReproducibleCmd.Flags().StringVarP(&rewritesFilepath, "rewrites", "", "", "A JSON file containing path rewrites") layersReproducibleCmd.Flags().StringVarP(&permsFilepath, "perms", "", "", "A JSON file containing file permissions") + layersReproducibleCmd.Flags().StringVarP(&historyFilepath, "history", "", "", "A JSON file containing layer history") layersReproducibleCmd.Flags().IntVarP(&maxLayers, "max-layers", "", 1, "The maximum number of layers") } diff --git a/cmd/utils.go b/cmd/utils.go index 068576c..ebb0403 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -5,6 +5,8 @@ import ( "os" "github.com/nlewo/nix2container/types" + + v1 "github.com/opencontainers/image-spec/specs-go/v1" ) func readPermsFile(filename string) (permPaths []types.PermPath, err error) { @@ -30,3 +32,15 @@ func readRewritesFile(filename string) (rewritePaths []types.RewritePath, err er } return } + +func readHistoryFile(filename string) (history v1.History, err error) { + content, err := os.ReadFile(filename) + if err != nil { + return history, err + } + err = json.Unmarshal(content, &history) + if err != nil { + return history, err + } + return +} diff --git a/default.nix b/default.nix index b0441e4..da0942e 100644 --- a/default.nix +++ b/default.nix @@ -247,6 +247,8 @@ let maxLayers ? 1, # Deprecated: will be removed on v1 contents ? null, + # Author, comment, created_by + metadata ? { created_by = "nix2container"; }, }: let subcommand = if reproducible then "layers-from-reproducible-storepaths" @@ -269,6 +271,8 @@ let rewritesFlag = "--rewrites ${rewritesFile}"; permsFile = pkgs.writeText "perms.json" (l.toJSON perms); permsFlag = l.optionalString (perms != []) "--perms ${permsFile}"; + historyFile = pkgs.writeText "history.json" (l.toJSON metadata); + historyFlag = l.optionalString (metadata != {}) "--history ${historyFile}"; allDeps = deps ++ copyToRootList; tarDirectory = l.optionalString (! reproducible) "--tar-directory $out"; layersJSON = pkgs.runCommand "layers.json" {} '' @@ -279,6 +283,7 @@ let --max-layers ${toString maxLayers} \ ${rewritesFlag} \ ${permsFlag} \ + ${historyFlag} \ ${tarDirectory} \ ${l.concatMapStringsSep " " (l: l + "/layers.json") layers} \ ''; diff --git a/examples/default.nix b/examples/default.nix index 104da7b..9b77596 100644 --- a/examples/default.nix +++ b/examples/default.nix @@ -15,4 +15,5 @@ nix-user = pkgs.callPackage ./nix-user.nix { inherit nix2container; }; ownership = pkgs.callPackage ./ownership.nix { inherit nix2container; }; created = pkgs.callPackage ./created.nix { inherit nix2container; }; + metadata = pkgs.callPackage ./metadata.nix { inherit nix2container; }; } diff --git a/examples/metadata.nix b/examples/metadata.nix new file mode 100644 index 0000000..ecf2860 --- /dev/null +++ b/examples/metadata.nix @@ -0,0 +1,17 @@ +{ pkgs, nix2container }: +nix2container.buildImage { + name = "metadata"; + config = { + entrypoint = ["${pkgs.hello}/bin/hello"]; + }; + layers = [ + (nix2container.buildLayer { + deps = [ pkgs.hello ]; + metadata = { + created_by = "test created_by"; + author = "test author"; + comment = "test comment"; + }; + }) + ]; +} diff --git a/nix/image.go b/nix/image.go index b9c7027..12527be 100644 --- a/nix/image.go +++ b/nix/image.go @@ -84,15 +84,14 @@ func getV1Image(image types.Image) (imageV1 v1.Image, err error) { imageV1.RootFS.DiffIDs, digest) imageV1.RootFS.Type = "layers" + // Even if optional in the spec, we + // need to add an history otherwise + // some toolings can complain: + // https://github.com/nlewo/nix2container/issues/57 imageV1.History = append( imageV1.History, - v1.History{ - // Even if optional in the spec, we - // need to add an history otherwise - // some toolings can complain: - // https://github.com/nlewo/nix2container/issues/57 - CreatedBy: "nix2container", - }) + layer.History, + ) } return } diff --git a/nix/image_test.go b/nix/image_test.go index f652474..d3c7485 100644 --- a/nix/image_test.go +++ b/nix/image_test.go @@ -38,6 +38,9 @@ func TestGetV1Image(t *testing.T) { DiffIDs: "sha256:adf74a52f9e1bcd7dab77193455fa06743b979cf5955148010e5becedba4f72d", Size: 10, MediaType: "application/vnd.oci.image.layer.v1.tar", + History: v1.History{ + CreatedBy: "nix2container", + }, }, }, } diff --git a/nix/layers.go b/nix/layers.go index 1bca326..7001779 100644 --- a/nix/layers.go +++ b/nix/layers.go @@ -64,7 +64,7 @@ func getPaths(storePaths []string, parents []types.Layer, rewrites []types.Rewri // If tarDirectory is not an empty string, the tar layer is written to // the disk. This is useful for layer containing non reproducible // store paths. -func newLayers(paths types.Paths, tarDirectory string, maxLayers int) (layers []types.Layer, err error) { +func newLayers(paths types.Paths, tarDirectory string, maxLayers int, history v1.History) (layers []types.Layer, err error) { offset := 0 for offset < len(paths) { max := offset + 1 @@ -90,6 +90,7 @@ func newLayers(paths types.Paths, tarDirectory string, maxLayers int) (layers [] Size: size, Paths: layerPaths, MediaType: v1.MediaTypeImageLayer, + History: history, } if tarDirectory != "" { // TODO: we should use v1.MediaTypeImageLayerGzip instead @@ -104,14 +105,14 @@ func newLayers(paths types.Paths, tarDirectory string, maxLayers int) (layers [] return layers, nil } -func NewLayers(storePaths []string, maxLayers int, parents []types.Layer, rewrites []types.RewritePath, exclude string, perms []types.PermPath) ([]types.Layer, error) { +func NewLayers(storePaths []string, maxLayers int, parents []types.Layer, rewrites []types.RewritePath, exclude string, perms []types.PermPath, history v1.History) ([]types.Layer, error) { paths := getPaths(storePaths, parents, rewrites, exclude, perms) - return newLayers(paths, "", maxLayers) + return newLayers(paths, "", maxLayers, history) } -func NewLayersNonReproducible(storePaths []string, maxLayers int, tarDirectory string, parents []types.Layer, rewrites []types.RewritePath, exclude string, perms []types.PermPath) (layers []types.Layer, err error) { +func NewLayersNonReproducible(storePaths []string, maxLayers int, tarDirectory string, parents []types.Layer, rewrites []types.RewritePath, exclude string, perms []types.PermPath, history v1.History) (layers []types.Layer, err error) { paths := getPaths(storePaths, parents, rewrites, exclude, perms) - return newLayers(paths, tarDirectory, maxLayers) + return newLayers(paths, tarDirectory, maxLayers, history) } func isPathInLayers(layers []types.Layer, path types.Path) bool { diff --git a/nix/layers_test.go b/nix/layers_test.go index 6816aa4..ef49360 100644 --- a/nix/layers_test.go +++ b/nix/layers_test.go @@ -5,6 +5,8 @@ import ( "github.com/nlewo/nix2container/types" "github.com/stretchr/testify/assert" + + v1 "github.com/opencontainers/image-spec/specs-go/v1" ) func TestPerms(t *testing.T) { @@ -18,7 +20,7 @@ func TestPerms(t *testing.T) { Mode: "0641", }, } - layer, err := NewLayers(paths, 1, []types.Layer{}, []types.RewritePath{}, "", perms) + layer, err := NewLayers(paths, 1, []types.Layer{}, []types.RewritePath{}, "", perms, v1.History{}) if err != nil { t.Fatalf("%v", err) } @@ -50,7 +52,7 @@ func TestNewLayers(t *testing.T) { paths := []string{ "../data/layer1/file1", } - layer, err := NewLayers(paths, 1, []types.Layer{}, []types.RewritePath{}, "", []types.PermPath{}) + layer, err := NewLayers(paths, 1, []types.Layer{}, []types.RewritePath{}, "", []types.PermPath{}, v1.History{}) if err != nil { t.Fatalf("%v", err) } @@ -70,7 +72,7 @@ func TestNewLayers(t *testing.T) { assert.Equal(t, expected, layer) tmpDir := t.TempDir() - layer, err = NewLayersNonReproducible(paths, 1, tmpDir, []types.Layer{}, []types.RewritePath{}, "", []types.PermPath{}) + layer, err = NewLayersNonReproducible(paths, 1, tmpDir, []types.Layer{}, []types.RewritePath{}, "", []types.PermPath{}, v1.History{}) if err != nil { t.Fatalf("%v", err) } diff --git a/tests/default.nix b/tests/default.nix index 755e8c8..b8ec34d 100644 --- a/tests/default.nix +++ b/tests/default.nix @@ -155,6 +155,45 @@ let exit $ret fi ''; + metadata = let + image = examples.metadata; + expected_created_by = "test created_by"; + expected_author = "test author"; + expected_comment = "test comment"; + in pkgs.writeScriptBin "test-script" '' + ${image.copyToPodman}/bin/copy-to-podman + created_by=$(${pkgs.podman}/bin/podman image inspect ${image.imageName}:${image.imageTag} -f '{{ (index .History 0).CreatedBy }}') + author=$(${pkgs.podman}/bin/podman image inspect ${image.imageName}:${image.imageTag} -f '{{ (index .History 0).Author }}') + comment=$(${pkgs.podman}/bin/podman image inspect ${image.imageName}:${image.imageTag} -f '{{ (index .History 0).Comment }}') + if ! echo $created_by | ${pkgs.gnugrep}/bin/grep '${expected_created_by}' > /dev/null; + then + echo "Expected created_by attribute to contain: ${expected_created_by}" + echo "" + echo "Actual created_by attribute: $created" + echo "" + echo "Error: test failed" + exit 1 + fi + if ! echo $author | ${pkgs.gnugrep}/bin/grep '${expected_author}' > /dev/null; + then + echo "Expected author attribute to contain: ${expected_author}" + echo "" + echo "Actual author attribute: $author" + echo "" + echo "Error: test failed" + exit 1 + fi + if ! echo $comment | ${pkgs.gnugrep}/bin/grep '${expected_comment}' > /dev/null; + then + echo "Expected comment attribute to contain: ${expected_comment}" + echo "" + echo "Actual comment attribute: $comment" + echo "" + echo "Error: test failed" + exit 1 + fi + echo "Test passed" + ''; } // (pkgs.lib.mapAttrs' (name: drv: { name = "${name}GetManifest"; @@ -178,4 +217,3 @@ let ${scripts} ''; in tests // { inherit all; } - diff --git a/types/types.go b/types/types.go index 66f3bed..0d27cb3 100644 --- a/types/types.go +++ b/types/types.go @@ -81,6 +81,7 @@ type Layer struct { // https://github.com/opencontainers/image-spec/blob/8b9d41f48198a7d6d0a5c1a12dc2d1f7f47fc97f/specs-go/v1/mediatype.go MediaType string `json:"mediatype"` LayerPath string `json:"layer-path,omitempty"` + History v1.History } func NewLayersFromFile(filename string) ([]Layer, error) {