diff --git a/pkg/image/disk.go b/pkg/image/disk.go index 21fc0d1582..89787738fc 100644 --- a/pkg/image/disk.go +++ b/pkg/image/disk.go @@ -90,6 +90,8 @@ func (img *DiskImage) InstantiateManifest(m *manifest.Manifest, for _, sysext := range img.Sysexts { var depsolveRef manifest.Pipeline + // Standalone extensions resolve and install their full package set + // into a fresh tree, without copying the OS tree or its RPM database. if !sysext.Standalone { depsolveRef = osPipeline } diff --git a/pkg/image/disk_test.go b/pkg/image/disk_test.go new file mode 100644 index 0000000000..f21166eb7d --- /dev/null +++ b/pkg/image/disk_test.go @@ -0,0 +1,94 @@ +package image_test + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/osbuild/image-builder/pkg/arch" + "github.com/osbuild/image-builder/pkg/depsolvednf" + "github.com/osbuild/image-builder/pkg/disk" + "github.com/osbuild/image-builder/pkg/image" + "github.com/osbuild/image-builder/pkg/manifest" + "github.com/osbuild/image-builder/pkg/osbuild" + "github.com/osbuild/image-builder/pkg/platform" + "github.com/osbuild/image-builder/pkg/rpmmd" + "github.com/osbuild/image-builder/pkg/runner" +) + +func TestDiskImageSysextStandalone(t *testing.T) { + for _, standalone := range []bool{false, true} { + name := "reuse OS" + if standalone { + name = "standalone" + } + t.Run(name, func(t *testing.T) { + img := image.NewDiskImage(&platform.Data{Arch: arch.ARCH_X86_64, ImageFormat: platform.FORMAT_RAW}, "disk.raw") + img.DiskCustomizations = manifest.NewDiskCustomizations() + img.PartitionTable = &disk.PartitionTable{ + Type: disk.PT_GPT, Size: 1024 * 1024 * 1024, + Partitions: []disk.Partition{{ + Start: 1024 * 1024, Size: 1023 * 1024 * 1024, + Payload: &disk.Filesystem{Type: "ext4", Mountpoint: "/", UUID: "11111111-1111-1111-1111-111111111111"}, + }}, + } + img.OSCustomizations.BasePackages = []string{"kernel"} + img.OSCustomizations.BaseRPMOptions = osbuild.RPMStageOptions{GPGKeysFromTree: []string{"/base-key"}} + img.Sysexts = []image.SysextConfig{{ + Name: "test", Format: "erofs", Standalone: standalone, + PackageSet: rpmmd.PackageSet{Include: []string{"extension"}}, + }} + mf := manifest.New() + _, err := img.InstantiateManifest(&mf, nil, &runner.Fedora{}, nil) + require.NoError(t, err) + chains, err := mf.GetPackageSetChains() + require.NoError(t, err) + if standalone { + require.Len(t, chains["sysext-test-tree"], 1) + assert.Equal(t, []string{"extension"}, chains["sysext-test-tree"][0].Include) + } else { + require.Len(t, chains["sysext-test-tree"], len(chains["os"])+1) + assert.Equal(t, chains["os"], chains["sysext-test-tree"][:len(chains["os"])]) + assert.Equal(t, []string{"extension"}, chains["sysext-test-tree"][len(chains["os"])].Include) + } + + depsolved := mockPackageSets() + installed := depsolved["os"].Transactions[0][0] + installed.Files = []string{"/base-key"} + depsolved["os"] = depsolvednf.DepsolveResult{Transactions: depsolvednf.TransactionList{{installed}}} + added := installed + added.Name = "extension" + added.Checksum.Value = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + added.Files = nil + transactions := depsolvednf.TransactionList{{installed}, {added}} + if standalone { + transactions = depsolvednf.TransactionList{{installed, added}} + } + depsolved["sysext-test-tree"] = depsolvednf.DepsolveResult{Transactions: transactions} + data, err := mf.Serialize(depsolved, nil, nil, nil, nil) + require.NoError(t, err) + pipeline := findPipelineFromOsbuildManifest(t, data, "sysext-test-tree") + require.NotNil(t, pipeline) + var want []*osbuild.Stage + packages := rpmmd.PackageList{added} + opts := &osbuild.RPMStageOptions{} + if standalone { + packages = rpmmd.PackageList{installed, added} + opts.GPGKeysFromTree = []string{"/base-key"} + } else { + want = append(want, osbuild.NewCopyStageSimple( + &osbuild.CopyStageOptions{Paths: []osbuild.CopyStagePath{{From: "input://tree/", To: "tree:///"}}}, + osbuild.NewPipelineTreeInputs("tree", "os"), + )) + } + want = append(want, osbuild.NewRPMStage(opts, osbuild.NewRpmStageSourceFilesInputs(packages))) + wantJSON, err := json.Marshal(want) + require.NoError(t, err) + gotJSON, err := json.Marshal(pipeline["stages"]) + require.NoError(t, err) + assert.JSONEq(t, string(wantJSON), string(gotJSON)) + }) + } +} diff --git a/pkg/manifest/sysext.go b/pkg/manifest/sysext.go index 936b63e725..c8c7791072 100644 --- a/pkg/manifest/sysext.go +++ b/pkg/manifest/sysext.go @@ -3,6 +3,7 @@ package manifest import ( "errors" "fmt" + "slices" "github.com/osbuild/image-builder/pkg/depsolvednf" "github.com/osbuild/image-builder/pkg/osbuild" @@ -32,9 +33,8 @@ type SysextTree struct { platform platform.Platform - // When non-nil, this pipeline's package sets are merged with the - // reference's sets so that shared dependencies resolve to the same - // versions as the base OS. + // When non-nil, copy this tree and install only packages not already + // present in it. Its package set chain precedes the extension transaction. depsolveReferencePipeline Pipeline Customizations SysextCustomizations @@ -78,8 +78,34 @@ func (p *SysextTree) serialize() (osbuild.Pipeline, error) { return osbuild.Pipeline{}, err } - // Do not copy the reference pipeline tree here; it causes issues with the RPM database and package selections. - rpmStages, err := osbuild.GenRPMStagesFromTransactions(p.depsolveResult.Transactions, p.Customizations.BaseRPMOptions) + transactions := p.depsolveResult.Transactions + baseRPMOptions := p.Customizations.BaseRPMOptions.Clone() + if ref := p.depsolveReferencePipeline; ref != nil { + // The reference transactions are already installed in the copied tree. + // The final transaction contains only the extension's additional packages. + if len(transactions) > 0 { + transactions = transactions[len(transactions)-1:] + } + pipeline.AddStage(osbuild.NewCopyStageSimple( + &osbuild.CopyStageOptions{Paths: []osbuild.CopyStagePath{{ + From: "input://tree/", + To: "tree:///", + }}}, + osbuild.NewPipelineTreeInputs("tree", ref.Name()), + )) + + // Keys provided by the reference packages have already been imported + // into the copied RPM database. Only import keys from new packages. + if baseRPMOptions != nil { + installed := ref.getPackageSpecs() + inheritedKeys := (depsolvednf.TransactionList{installed}).GetFilesTransactionInfo(baseRPMOptions.GPGKeysFromTree) + baseRPMOptions.GPGKeysFromTree = slices.DeleteFunc(baseRPMOptions.GPGKeysFromTree, func(path string) bool { + _, inherited := inheritedKeys[path] + return inherited + }) + } + } + rpmStages, err := osbuild.GenRPMStagesFromTransactions(transactions, baseRPMOptions) if err != nil { return osbuild.Pipeline{}, err } @@ -94,20 +120,22 @@ func (p *SysextTree) getPackageSetChain(d Distro) ([]rpmmd.PackageSet, error) { return nil, nil } + pkgSet.Repositories = slices.Concat(pkgSet.Repositories, p.depsolveRepos) + pkgSet.InstallWeakDeps = false + var chain []rpmmd.PackageSet if p.depsolveReferencePipeline != nil { - refChain, err := p.depsolveReferencePipeline.getPackageSetChain(d) + var err error + chain, err = p.depsolveReferencePipeline.getPackageSetChain(d) if err != nil { return nil, fmt.Errorf("getting reference package set chain: %w", err) } - for _, refSet := range refChain { - pkgSet = pkgSet.Append(refSet) - pkgSet.Repositories = append(pkgSet.Repositories, refSet.Repositories...) + if len(chain) > 0 { + // Each transaction must retain its predecessor's repositories in order. + pkgSet.Repositories = slices.Concat(chain[len(chain)-1].Repositories, pkgSet.Repositories) } } - pkgSet.Repositories = append(pkgSet.Repositories, p.depsolveRepos...) - pkgSet.InstallWeakDeps = false - return []rpmmd.PackageSet{pkgSet}, nil + return append(chain, pkgSet), nil } // SysextPrep computes the file-level difference between a reference tree @@ -186,8 +214,8 @@ type SysextPipelines struct { // NewSysextPipelines creates the full Tree -> Prep chain for a named sysext. // filesystemReferencePipeline is used by the Prep stage to compute the // tree-delta and is always required. depsolveReferencePipeline is optional: -// when non-nil, its package sets are merged into the tree's depsolve so that -// shared dependencies resolve to the same versions as the base OS. +// when non-nil, its tree is copied and its package set chain precedes the +// extension's own transaction. Only the final transaction is installed. func NewSysextPipelines(buildPipeline Build, platform platform.Platform, repos []rpmmd.RepoConfig, filesystemReferencePipeline Pipeline, depsolveReferencePipeline Pipeline, name string) SysextPipelines { tree := NewSysextTree(buildPipeline, platform, repos, name) tree.depsolveReferencePipeline = depsolveReferencePipeline diff --git a/pkg/manifest/sysext_test.go b/pkg/manifest/sysext_test.go index 7ec32e7ad9..5d4f8ad87f 100644 --- a/pkg/manifest/sysext_test.go +++ b/pkg/manifest/sysext_test.go @@ -1,6 +1,7 @@ package manifest_test import ( + "encoding/json" "testing" "github.com/stretchr/testify/assert" @@ -75,6 +76,179 @@ func TestSysextTreeSerializeBaseRPMOptions(t *testing.T) { assert.Equal(t, []string{"en_US"}, opts.InstallLangs) } +func TestSysextTreeReuseOS(t *testing.T) { + installed := testInputs().Depsolved.Transactions[0][0] + first := installed + first.Name = "first" + first.Checksum.Value = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + second := installed + second.Name = "second" + second.Checksum.Value = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + + tests := []struct { + name string + transactions depsolvednf.TransactionList + standalone bool + want depsolvednf.TransactionList + }{ + { + name: "only install extension transaction after multiple reference transactions", + transactions: depsolvednf.TransactionList{{installed}, {first}, {second}}, + want: depsolvednf.TransactionList{{second}}, + }, + { + name: "all packages already installed", + transactions: depsolvednf.TransactionList{{installed}, {}}, + }, + { + name: "no packages requested", + }, + { + name: "standalone installs full transaction without copying", + standalone: true, + transactions: depsolvednf.TransactionList{{installed, first}}, + want: depsolvednf.TransactionList{{installed, first}}, + }, + { + name: "standalone installs all transactions without copying", + standalone: true, + transactions: depsolvednf.TransactionList{{installed}, {first, second}}, + want: depsolvednf.TransactionList{{installed}, {first, second}}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + mani := manifest.New() + build := manifest.NewContentTestBuild(&mani, nil, nil, nil) + ref := manifest.NewContentTest("os", build, nil, nil, nil) + _, err := manifest.SerializeWith(ref, manifest.Inputs{Depsolved: depsolvednf.DepsolveResult{ + Transactions: depsolvednf.TransactionList{{installed}}, + }}) + require.NoError(t, err) + var depsolveRef manifest.Pipeline = ref + if tc.standalone { + depsolveRef = nil + } + sp := manifest.NewSysextPipelines(build, nil, nil, ref, depsolveRef, "test") + before, err := json.Marshal(tc.transactions) + require.NoError(t, err) + pipeline, err := manifest.SerializeWith(sp.Tree, manifest.Inputs{Depsolved: depsolvednf.DepsolveResult{ + Transactions: tc.transactions, + }}) + require.NoError(t, err) + after, err := json.Marshal(tc.transactions) + require.NoError(t, err) + assert.Equal(t, before, after, "serialization must not modify the depsolve result") + + stages := pipeline.Stages + if !tc.standalone { + require.NotEmpty(t, stages) + assert.Equal(t, osbuild.NewCopyStageSimple( + &osbuild.CopyStageOptions{Paths: []osbuild.CopyStagePath{{From: "input://tree/", To: "tree:///"}}}, + osbuild.NewPipelineTreeInputs("tree", "os"), + ), stages[0]) + stages = stages[1:] + } + require.Len(t, stages, len(tc.want)) + for i, pkgs := range tc.want { + assert.Equal(t, "org.osbuild.rpm", stages[i].Type) + assert.Equal(t, osbuild.NewRpmStageSourceFilesInputs(pkgs), stages[i].Inputs) + options, err := json.Marshal(stages[i].Options) + require.NoError(t, err) + assert.NotContains(t, string(options), "nodeps") + } + }) + } +} + +func TestSysextTreeReuseOSInheritedGPGKeys(t *testing.T) { + mani := manifest.New() + build := manifest.NewContentTestBuild(&mani, nil, nil, nil) + ref := manifest.NewContentTest("os", build, nil, nil, nil) + installed := testInputs().Depsolved.Transactions[0][0] + installed.Files = []string{"/base-key"} + _, err := manifest.SerializeWith(ref, manifest.Inputs{Depsolved: depsolvednf.DepsolveResult{ + Transactions: depsolvednf.TransactionList{{installed}}, + }}) + require.NoError(t, err) + added := installed + added.Name = "extension" + added.Checksum.Value = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + added.Files = []string{"/extension-key"} + sp := manifest.NewSysextPipelines(build, nil, nil, ref, ref, "test") + sp.Tree.Customizations.BaseRPMOptions = &osbuild.RPMStageOptions{ + GPGKeysFromTree: []string{"/base-key", "/extension-key"}, + InstallLangs: []string{"en_US"}, + DisableDracut: true, + } + pipeline, err := manifest.SerializeWith(sp.Tree, manifest.Inputs{Depsolved: depsolvednf.DepsolveResult{ + Transactions: depsolvednf.TransactionList{{installed}, {added}}, + }}) + require.NoError(t, err) + require.Len(t, pipeline.Stages, 2) + opts := pipeline.Stages[1].Options.(*osbuild.RPMStageOptions) + assert.Equal(t, []string{"/extension-key"}, opts.GPGKeysFromTree) + assert.Equal(t, []string{"en_US"}, opts.InstallLangs) + assert.True(t, opts.DisableDracut) + assert.Equal(t, []string{"/base-key", "/extension-key"}, sp.Tree.Customizations.BaseRPMOptions.GPGKeysFromTree) +} + +func TestSysextTreeReuseOSManifest(t *testing.T) { + mani := manifest.New() + build := manifest.NewContentTestBuild(&mani, nil, nil, nil) + pf := &platform.Data{Arch: arch.ARCH_X86_64} + ref := manifest.NewOS(build, pf, nil) + sp := manifest.NewSysextPipelines(build, pf, nil, ref, ref, "test") + baseInputs := testInputs() + baseInputs.Depsolved.Transactions[0][0].RemoteLocations = []string{"https://example.com/base.rpm"} + added := baseInputs.Depsolved.Transactions[0][0] + added.Name = "extension" + added.RemoteLocations = []string{"https://example.com/extension.rpm"} + added.Checksum.Value = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + depsolved := map[string]depsolvednf.DepsolveResult{ + ref.Name(): baseInputs.Depsolved, + sp.Tree.Name(): { + Transactions: depsolvednf.TransactionList{baseInputs.Depsolved.Transactions[0], {added}}, + }, + } + // Repeated serialization must not consume or alter either pipeline's inputs. + var previous manifest.OSBuildManifest + for range 2 { + data, err := mani.Serialize(depsolved, nil, nil, nil, nil) + require.NoError(t, err) + if previous != nil { + assert.Equal(t, previous, data) + } + previous = data + type stage struct { + Type string `json:"type"` + Inputs json.RawMessage `json:"inputs"` + } + var result struct { + Pipelines []struct { + Name string `json:"name"` + Stages []stage `json:"stages"` + } `json:"pipelines"` + } + require.NoError(t, json.Unmarshal(data, &result)) + var stages []stage + for _, pipeline := range result.Pipelines { + if pipeline.Name == sp.Tree.Name() { + stages = pipeline.Stages + } + } + require.Len(t, stages, 2) + assert.Equal(t, "org.osbuild.copy", stages[0].Type) + copyInputs, err := json.Marshal(osbuild.NewPipelineTreeInputs("tree", ref.Name())) + require.NoError(t, err) + assert.JSONEq(t, string(copyInputs), string(stages[0].Inputs)) + assert.Equal(t, "org.osbuild.rpm", stages[1].Type) + rpmInputs, err := json.Marshal(osbuild.NewRpmStageSourceFilesInputs(rpmmd.PackageList{added})) + require.NoError(t, err) + assert.JSONEq(t, string(rpmInputs), string(stages[1].Inputs)) + } +} + func TestSysextTreeSerializeNotStarted(t *testing.T) { base, _ := newTestSysextTree(t) _, err := manifest.Serialize(base) @@ -274,27 +448,49 @@ func TestNewSysextPipelines(t *testing.T) { assert.Equal(t, "sysext-myext-prep", pipelines.Prep.Name()) } -func TestSysextTreeGetPackageSetChainMergesReference(t *testing.T) { +func TestSysextTreeGetPackageSetChainPreservesReference(t *testing.T) { mani := manifest.New() r := &runner.Linux{} build := manifest.NewBuild(&mani, r, nil, nil) pf := &platform.Data{Arch: arch.ARCH_X86_64} + baseRepo := rpmmd.RepoConfig{Id: "base"} + payloadRepo := rpmmd.RepoConfig{Id: "payload"} + extensionRepo := rpmmd.RepoConfig{Id: "extension"} - refPipeline := manifest.NewOS(build, pf, nil) + refPipeline := manifest.NewOS(build, pf, []rpmmd.RepoConfig{baseRepo}) refPipeline.OSCustomizations.BasePackages = []string{"coreutils", "bash"} + refPipeline.OSCustomizations.ExcludeBasePackages = []string{"excluded"} + refPipeline.OSCustomizations.InstallWeakDeps = true + refPipeline.OSCustomizations.LockRootUser = true + refPipeline.OSCustomizations.BlueprintPackages = []string{"blueprint"} + refPipeline.OSCustomizations.BlueprintModules = []string{"test:stream"} + refPipeline.OSCustomizations.PayloadRepos = []rpmmd.RepoConfig{payloadRepo} + refChain, err := refPipeline.GetPackageSetChain(0) + require.NoError(t, err) + require.Len(t, refChain, 3) - sp := manifest.NewSysextPipelines(build, pf, nil, refPipeline, refPipeline, "nginx") - sp.Tree.Customizations.PackageSet = rpmmd.PackageSet{ - Include: []string{"nginx"}, - } + for _, name := range []string{"nginx", "tmux"} { + sp := manifest.NewSysextPipelines(build, pf, []rpmmd.RepoConfig{extensionRepo}, refPipeline, refPipeline, name) + sp.Tree.Customizations.PackageSet = rpmmd.PackageSet{ + Include: []string{name}, + Exclude: []string{"extension-excluded"}, + InstallWeakDeps: true, + } - chain, err := sp.Tree.GetPackageSetChain(0) + chain, err := sp.Tree.GetPackageSetChain(0) + require.NoError(t, err) + require.Len(t, chain, len(refChain)+1) + assert.Equal(t, refChain, chain[:len(refChain)]) + assert.Equal(t, rpmmd.PackageSet{ + Include: []string{name}, + Exclude: []string{"extension-excluded"}, + Repositories: []rpmmd.RepoConfig{baseRepo, payloadRepo, extensionRepo}, + }, chain[len(refChain)]) + } + // Constructing extension chains must not change the reference's inputs. + after, err := refPipeline.GetPackageSetChain(0) require.NoError(t, err) - require.Len(t, chain, 1) - assert.Contains(t, chain[0].Include, "nginx") - assert.Contains(t, chain[0].Include, "coreutils") - assert.Contains(t, chain[0].Include, "bash") - assert.False(t, chain[0].InstallWeakDeps) + assert.Equal(t, refChain, after) } func TestSysextTreeGetPackageSetChainNilDepsolveReference(t *testing.T) {