new: added a publish s3 subcmd, used by build cmd.

Signed-off-by: Federico Di Pierro <[email protected]>

.github/workflows/ci.yml

@@ -30,6 +30,7 @@ jobs:
       - name: Test
         run: make test
 
-      - name: Update coverage report
-        if: github.event_name == 'push'
-        uses: ncruces/go-coverage-report@v0
+      # NOT WORKING FOR NOW because of build constraints
+      #- name: Update coverage report
+      #  if: github.event_name == 'push'
+      #  uses: ncruces/go-coverage-report@v0

README.md

@@ -19,6 +19,7 @@ Right now, the tool implements, under the `configs` subcmd:
 Moreover, under the `s3` subcmd:
 * s3 driver stats
 * s3 driver cleanup
+* s3 driver publish
 
 This is enough to port [`update-dbg` image](https://github.com/falcosecurity/test-infra/tree/master/images/update-dbg) to make use of this tool instead of the currently used bash scripts.  
 First benchmarks showed a tremendous perf improvement: old update-dbg scripts took around 50m on my laptop for a single driverversion. The new tool takes ~10s.  

cmd/publish/publish.go

@@ -0,0 +1,29 @@
+package publish
+
+import (
+	"github.com/fededp/dbg-go/pkg/publish"
+	"github.com/fededp/dbg-go/pkg/root"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+func NewPublishCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "publish",
+		Short: "publish local drivers to remote bucket",
+		RunE:  execute,
+	}
+	flags := cmd.Flags()
+	flags.String("aws-profile", "", "aws-profile to be used. Mandatory.")
+
+	_ = cmd.MarkFlagRequired("aws-profile")
+	return cmd
+}
+
+func execute(_ *cobra.Command, _ []string) error {
+	options := publish.Options{
+		Options:    root.LoadRootOptions(),
+		AwsProfile: viper.GetString("aws-profile"),
+	}
+	return publish.Run(options)
+}

cmd/s3.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"github.com/fededp/dbg-go/cmd/cleanup"
+	"github.com/fededp/dbg-go/cmd/publish"
 	"github.com/fededp/dbg-go/cmd/stats"
 	"github.com/spf13/cobra"
 )
@@ -17,4 +18,5 @@ func init() {
 	// Subcommands
 	s3Cmd.AddCommand(cleanup.NewCleanupS3Cmd())
 	s3Cmd.AddCommand(stats.NewStatsS3Cmd())
+	s3Cmd.AddCommand(publish.NewPublishCmd())
 }

pkg/build/build.go

@@ -3,6 +3,7 @@ package build
 import (
 	"github.com/falcosecurity/driverkit/cmd"
 	"github.com/falcosecurity/driverkit/pkg/driverbuilder"
+	"github.com/fededp/dbg-go/pkg/publish"
 	"github.com/fededp/dbg-go/pkg/root"
 	s3utils "github.com/fededp/dbg-go/pkg/utils/s3"
 	"github.com/fededp/dbg-go/pkg/validate"
@@ -13,6 +14,7 @@ import (
 	"path/filepath"
 )
 
+// Used by tests
 var testClient *s3utils.Client
 
 func Run(opts Options) error {
@@ -92,29 +94,9 @@ func buildConfig(client *s3utils.Client, opts Options, driverVersion, configPath
 	}
 
 	if opts.Publish {
-		logger.Info("publishing")
-		// Publish object!
-		if ro.Output.Module != "" {
-			f, err := os.Open(ro.Output.Module)
-			if err != nil {
-				return err
-			}
-			err = client.PutObject(opts.Options, driverVersion, filepath.Base(ro.Output.Module), f)
-			_ = f.Close()
-			if err != nil {
-				return err
-			}
-		}
-		if ro.Output.Probe != "" {
-			f, err := os.Open(ro.Output.Probe)
-			if err != nil {
-				return err
-			}
-			err = client.PutObject(opts.Options, driverVersion, filepath.Base(ro.Output.Probe), f)
-			_ = f.Close()
-			if err != nil {
-				return err
-			}
+		err = publish.LoopDriversFiltered(client, opts.Options)
+		if err != nil {
+			return err
 		}
 	}
 	return nil

pkg/build/build_test.go

@@ -16,7 +16,7 @@ import (
 	"time"
 )
 
-// NOTE: this test might be flaking because it tries to build some configs against a driverversion
+// NOTE: this test might be flaking because it tries to build some configs against a driver version.
 // When it fails, just update configs to be built.
 func TestBuild(t *testing.T) {
 	if runtime.GOARCH != "amd64" {

pkg/publish/publish.go

@@ -0,0 +1,60 @@
+package publish
+
+import (
+	"fmt"
+	"github.com/fededp/dbg-go/pkg/root"
+	s3utils "github.com/fededp/dbg-go/pkg/utils/s3"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// Used by tests
+var testClient *s3utils.Client
+
+func Run(opts Options) error {
+	slog.Info("publishing drivers")
+	var (
+		client *s3utils.Client
+		err    error
+	)
+	if testClient == nil {
+		client, err = s3utils.NewClient(false, opts.AwsProfile)
+		if err != nil {
+			return err
+		}
+	} else {
+		client = testClient
+	}
+	return LoopDriversFiltered(client, opts.Options)
+}
+
+func LoopDriversFiltered(client *s3utils.Client, opts root.Options) error {
+	outputNameGlob := opts.Target.ToGlob()
+	outputNameGlob = strings.ReplaceAll(outputNameGlob, ".yaml", ".*")
+	for _, driverVersion := range opts.DriverVersion {
+		driversPath := fmt.Sprintf("output/%s/%s/falco_%s", driverVersion, opts.Architecture.ToNonDeb(), outputNameGlob)
+		drivers, err := filepath.Glob(driversPath)
+		if err != nil {
+			return err
+		}
+		for _, driver := range drivers {
+			slog.Info("publishing", "driver", driver)
+			if opts.DryRun {
+				slog.Info("skipping because of dry-run.")
+				return nil
+			}
+			f, err := os.Open(driver)
+			if err != nil {
+				return err
+			}
+			err = client.PutObject(opts, driverVersion, filepath.Base(driver), f)
+			_ = f.Close()
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

pkg/publish/publish_test.go

@@ -0,0 +1,84 @@
+package publish
+
+import (
+	"context"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/fededp/dbg-go/pkg/root"
+	s3utils "github.com/fededp/dbg-go/pkg/utils/s3"
+	testutils "github.com/fededp/dbg-go/pkg/utils/test"
+	"github.com/stretchr/testify/assert"
+	"os"
+	"testing"
+)
+
+func TestPublish(t *testing.T) {
+	// This client will be used by the Run action
+	testClient = testutils.S3CreateTestBucket(t, nil)
+
+	// Create output folder
+	err := os.MkdirAll("output/5.0.1+driver/x86_64", 0700)
+	assert.NoError(t, err)
+	t.Cleanup(func() {
+		_ = os.RemoveAll("output")
+	})
+
+	// Create a fake kernel module object to be uploaded to test bucket
+	d1 := []byte("TEST\n")
+	err = os.WriteFile("output/5.0.1+driver/x86_64/falco_almalinux_4.18.0-425.10.1.el8_7.x86_64_1.ko", d1, 0644)
+	assert.NoError(t, err)
+
+	// Fetch an existing object metadata
+	realClient, err := s3utils.NewClient(true, "")
+	assert.NoError(t, err)
+	object, err := realClient.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(s3utils.S3Bucket),
+		Key:    aws.String("driver/5.0.1+driver/x86_64/falco_almalinux_4.18.0-425.10.1.el8_7.x86_64_1.ko"),
+	})
+	assert.NoError(t, err)
+
+	// Run our action to upload our driver object
+	err = Run(Options{
+		Options: root.Options{
+			RepoRoot:      "./test",
+			Architecture:  "amd64",
+			DriverName:    "falco",
+			DriverVersion: []string{"5.0.1+driver"},
+			Target: root.Target{
+				Distro:        "almalinux",
+				KernelRelease: "4.18.0-425*",
+				KernelVersion: "1",
+			},
+		},
+	})
+	assert.NoError(t, err)
+
+	// Fetch test object metadata
+	testObject, err := testClient.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(s3utils.S3Bucket),
+		Key:    aws.String("driver/5.0.1+driver/x86_64/falco_almalinux_4.18.0-425.10.1.el8_7.x86_64_1.ko"),
+	})
+
+	// Check that published object has correct metadata
+	assert.Equal(t, testObject.ServerSideEncryption, object.ServerSideEncryption)
+	assert.Equal(t, testObject.AcceptRanges, object.AcceptRanges)
+	assert.Equal(t, testObject.ObjectLockMode, object.ObjectLockMode)
+	assert.Equal(t, testObject.ArchiveStatus, object.ArchiveStatus)
+	assert.Equal(t, testObject.BucketKeyEnabled, object.BucketKeyEnabled)
+	assert.Equal(t, testObject.CacheControl, object.CacheControl)
+	assert.Equal(t, testObject.ContentDisposition, object.ContentDisposition)
+	assert.Equal(t, testObject.ContentEncoding, object.ContentEncoding)
+	assert.Equal(t, testObject.ContentType, object.ContentType)
+	assert.Equal(t, testObject.Expiration, object.Expiration)
+	assert.Equal(t, testObject.Expires, object.Expires)
+	assert.Equal(t, testObject.Metadata, object.Metadata)
+	assert.Equal(t, testObject.DeleteMarker, object.DeleteMarker)
+	assert.Equal(t, testObject.MissingMeta, object.MissingMeta)
+	assert.Equal(t, testObject.ObjectLockLegalHoldStatus, object.ObjectLockLegalHoldStatus)
+	assert.Equal(t, testObject.ObjectLockRetainUntilDate, object.ObjectLockRetainUntilDate)
+	assert.Equal(t, testObject.PartsCount, object.PartsCount)
+	assert.Equal(t, testObject.ReplicationStatus, object.ReplicationStatus)
+	assert.Equal(t, testObject.RequestCharged, object.RequestCharged)
+	assert.Equal(t, testObject.Restore, object.Restore)
+	assert.Equal(t, testObject.StorageClass, object.StorageClass)
+}

pkg/publish/types.go

@@ -0,0 +1,8 @@
+package publish
+
+import "github.com/fededp/dbg-go/pkg/root"
+
+type Options struct {
+	root.Options
+	AwsProfile string
+}