diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b3a5a66..05b6b64 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,7 @@ jobs: - name: Test run: make test - - name: Update coverage report - if: github.event_name == 'push' - uses: ncruces/go-coverage-report@v0 \ No newline at end of file + # NOT WORKING FOR NOW because of build constraints + #- name: Update coverage report + # if: github.event_name == 'push' + # uses: ncruces/go-coverage-report@v0 \ No newline at end of file diff --git a/README.md b/README.md index 8d54d36..902fe36 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ Right now, the tool implements, under the `configs` subcmd: Moreover, under the `s3` subcmd: * s3 driver stats * s3 driver cleanup +* s3 driver publish This is enough to port [`update-dbg` image](https://github.com/falcosecurity/test-infra/tree/master/images/update-dbg) to make use of this tool instead of the currently used bash scripts. First benchmarks showed a tremendous perf improvement: old update-dbg scripts took around 50m on my laptop for a single driverversion. The new tool takes ~10s. diff --git a/cmd/publish/publish.go b/cmd/publish/publish.go new file mode 100644 index 0000000..b1ce4c3 --- /dev/null +++ b/cmd/publish/publish.go @@ -0,0 +1,29 @@ +package publish + +import ( + "github.com/fededp/dbg-go/pkg/publish" + "github.com/fededp/dbg-go/pkg/root" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +func NewPublishCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "publish", + Short: "publish local drivers to remote bucket", + RunE: execute, + } + flags := cmd.Flags() + flags.String("aws-profile", "", "aws-profile to be used. Mandatory.") + + _ = cmd.MarkFlagRequired("aws-profile") + return cmd +} + +func execute(_ *cobra.Command, _ []string) error { + options := publish.Options{ + Options: root.LoadRootOptions(), + AwsProfile: viper.GetString("aws-profile"), + } + return publish.Run(options) +} diff --git a/cmd/s3.go b/cmd/s3.go index e152b62..b3077b9 100644 --- a/cmd/s3.go +++ b/cmd/s3.go @@ -2,6 +2,7 @@ package cmd import ( "github.com/fededp/dbg-go/cmd/cleanup" + "github.com/fededp/dbg-go/cmd/publish" "github.com/fededp/dbg-go/cmd/stats" "github.com/spf13/cobra" ) @@ -17,4 +18,5 @@ func init() { // Subcommands s3Cmd.AddCommand(cleanup.NewCleanupS3Cmd()) s3Cmd.AddCommand(stats.NewStatsS3Cmd()) + s3Cmd.AddCommand(publish.NewPublishCmd()) } diff --git a/pkg/build/build.go b/pkg/build/build.go index 036a745..2d3f6af 100644 --- a/pkg/build/build.go +++ b/pkg/build/build.go @@ -3,6 +3,7 @@ package build import ( "github.com/falcosecurity/driverkit/cmd" "github.com/falcosecurity/driverkit/pkg/driverbuilder" + "github.com/fededp/dbg-go/pkg/publish" "github.com/fededp/dbg-go/pkg/root" s3utils "github.com/fededp/dbg-go/pkg/utils/s3" "github.com/fededp/dbg-go/pkg/validate" @@ -13,6 +14,7 @@ import ( "path/filepath" ) +// Used by tests var testClient *s3utils.Client func Run(opts Options) error { @@ -92,29 +94,9 @@ func buildConfig(client *s3utils.Client, opts Options, driverVersion, configPath } if opts.Publish { - logger.Info("publishing") - // Publish object! - if ro.Output.Module != "" { - f, err := os.Open(ro.Output.Module) - if err != nil { - return err - } - err = client.PutObject(opts.Options, driverVersion, filepath.Base(ro.Output.Module), f) - _ = f.Close() - if err != nil { - return err - } - } - if ro.Output.Probe != "" { - f, err := os.Open(ro.Output.Probe) - if err != nil { - return err - } - err = client.PutObject(opts.Options, driverVersion, filepath.Base(ro.Output.Probe), f) - _ = f.Close() - if err != nil { - return err - } + err = publish.LoopDriversFiltered(client, opts.Options) + if err != nil { + return err } } return nil diff --git a/pkg/build/build_test.go b/pkg/build/build_test.go index 5806d67..2d066ec 100644 --- a/pkg/build/build_test.go +++ b/pkg/build/build_test.go @@ -16,7 +16,7 @@ import ( "time" ) -// NOTE: this test might be flaking because it tries to build some configs against a driverversion +// NOTE: this test might be flaking because it tries to build some configs against a driver version. // When it fails, just update configs to be built. func TestBuild(t *testing.T) { if runtime.GOARCH != "amd64" { diff --git a/pkg/publish/publish.go b/pkg/publish/publish.go new file mode 100644 index 0000000..1a4ff88 --- /dev/null +++ b/pkg/publish/publish.go @@ -0,0 +1,60 @@ +package publish + +import ( + "fmt" + "github.com/fededp/dbg-go/pkg/root" + s3utils "github.com/fededp/dbg-go/pkg/utils/s3" + "log/slog" + "os" + "path/filepath" + "strings" +) + +// Used by tests +var testClient *s3utils.Client + +func Run(opts Options) error { + slog.Info("publishing drivers") + var ( + client *s3utils.Client + err error + ) + if testClient == nil { + client, err = s3utils.NewClient(false, opts.AwsProfile) + if err != nil { + return err + } + } else { + client = testClient + } + return LoopDriversFiltered(client, opts.Options) +} + +func LoopDriversFiltered(client *s3utils.Client, opts root.Options) error { + outputNameGlob := opts.Target.ToGlob() + outputNameGlob = strings.ReplaceAll(outputNameGlob, ".yaml", ".*") + for _, driverVersion := range opts.DriverVersion { + driversPath := fmt.Sprintf("output/%s/%s/falco_%s", driverVersion, opts.Architecture.ToNonDeb(), outputNameGlob) + drivers, err := filepath.Glob(driversPath) + if err != nil { + return err + } + for _, driver := range drivers { + slog.Info("publishing", "driver", driver) + if opts.DryRun { + slog.Info("skipping because of dry-run.") + return nil + } + f, err := os.Open(driver) + if err != nil { + return err + } + err = client.PutObject(opts, driverVersion, filepath.Base(driver), f) + _ = f.Close() + if err != nil { + return err + } + } + } + return nil +} diff --git a/pkg/publish/publish_test.go b/pkg/publish/publish_test.go new file mode 100644 index 0000000..be4dfde --- /dev/null +++ b/pkg/publish/publish_test.go @@ -0,0 +1,84 @@ +package publish + +import ( + "context" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/fededp/dbg-go/pkg/root" + s3utils "github.com/fededp/dbg-go/pkg/utils/s3" + testutils "github.com/fededp/dbg-go/pkg/utils/test" + "github.com/stretchr/testify/assert" + "os" + "testing" +) + +func TestPublish(t *testing.T) { + // This client will be used by the Run action + testClient = testutils.S3CreateTestBucket(t, nil) + + // Create output folder + err := os.MkdirAll("output/5.0.1+driver/x86_64", 0700) + assert.NoError(t, err) + t.Cleanup(func() { + _ = os.RemoveAll("output") + }) + + // Create a fake kernel module object to be uploaded to test bucket + d1 := []byte("TEST\n") + err = os.WriteFile("output/5.0.1+driver/x86_64/falco_almalinux_4.18.0-425.10.1.el8_7.x86_64_1.ko", d1, 0644) + assert.NoError(t, err) + + // Fetch an existing object metadata + realClient, err := s3utils.NewClient(true, "") + assert.NoError(t, err) + object, err := realClient.HeadObject(context.Background(), &s3.HeadObjectInput{ + Bucket: aws.String(s3utils.S3Bucket), + Key: aws.String("driver/5.0.1+driver/x86_64/falco_almalinux_4.18.0-425.10.1.el8_7.x86_64_1.ko"), + }) + assert.NoError(t, err) + + // Run our action to upload our driver object + err = Run(Options{ + Options: root.Options{ + RepoRoot: "./test", + Architecture: "amd64", + DriverName: "falco", + DriverVersion: []string{"5.0.1+driver"}, + Target: root.Target{ + Distro: "almalinux", + KernelRelease: "4.18.0-425*", + KernelVersion: "1", + }, + }, + }) + assert.NoError(t, err) + + // Fetch test object metadata + testObject, err := testClient.HeadObject(context.Background(), &s3.HeadObjectInput{ + Bucket: aws.String(s3utils.S3Bucket), + Key: aws.String("driver/5.0.1+driver/x86_64/falco_almalinux_4.18.0-425.10.1.el8_7.x86_64_1.ko"), + }) + + // Check that published object has correct metadata + assert.Equal(t, testObject.ServerSideEncryption, object.ServerSideEncryption) + assert.Equal(t, testObject.AcceptRanges, object.AcceptRanges) + assert.Equal(t, testObject.ObjectLockMode, object.ObjectLockMode) + assert.Equal(t, testObject.ArchiveStatus, object.ArchiveStatus) + assert.Equal(t, testObject.BucketKeyEnabled, object.BucketKeyEnabled) + assert.Equal(t, testObject.CacheControl, object.CacheControl) + assert.Equal(t, testObject.ContentDisposition, object.ContentDisposition) + assert.Equal(t, testObject.ContentEncoding, object.ContentEncoding) + assert.Equal(t, testObject.ContentType, object.ContentType) + assert.Equal(t, testObject.Expiration, object.Expiration) + assert.Equal(t, testObject.Expires, object.Expires) + assert.Equal(t, testObject.Metadata, object.Metadata) + assert.Equal(t, testObject.DeleteMarker, object.DeleteMarker) + assert.Equal(t, testObject.MissingMeta, object.MissingMeta) + assert.Equal(t, testObject.ObjectLockLegalHoldStatus, object.ObjectLockLegalHoldStatus) + assert.Equal(t, testObject.ObjectLockRetainUntilDate, object.ObjectLockRetainUntilDate) + assert.Equal(t, testObject.PartsCount, object.PartsCount) + assert.Equal(t, testObject.ReplicationStatus, object.ReplicationStatus) + assert.Equal(t, testObject.RequestCharged, object.RequestCharged) + assert.Equal(t, testObject.Restore, object.Restore) + assert.Equal(t, testObject.StorageClass, object.StorageClass) +} diff --git a/pkg/publish/types.go b/pkg/publish/types.go new file mode 100644 index 0000000..254adfa --- /dev/null +++ b/pkg/publish/types.go @@ -0,0 +1,8 @@ +package publish + +import "github.com/fededp/dbg-go/pkg/root" + +type Options struct { + root.Options + AwsProfile string +}