feat: dws-2909 bucket size init

.github/workflows/ci.yml

@@ -18,7 +18,7 @@ jobs:
         uses: actions/checkout@v3
       - uses: actions/setup-go@v4
         with:
-          go-version: '1.23.0'
+          go-version: '1.23.3'
           cache: false
           args: --timeout=10m -v
 
@@ -35,7 +35,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: '1.23.0'
+          go-version: '1.23.3'
           cache: false
 
       - name: Checkout code

Makefile

@@ -381,3 +381,7 @@ integration-services-start:
 		--minio.access-key "$$AWS_ACCESS_KEY_ID" \
 		--minio.secret-key "$$AWS_SECRET_ACCESS_KEY" \
 		--s3.fully-compatible-listing
+
+.PHONY: vendor
+vendor:
+	go mod tidy && go mod vendor

go.mod

@@ -1,12 +1,13 @@
 module github.com/deweb-services/gateway-st
 
-go 1.23.0
-
-toolchain go1.23.2
+go 1.23.3
 
 require (
+	github.com/deweb-services/go-nodeshift v0.0.2
+	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/minio/cli v1.22.0
 	github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78
+	github.com/sirupsen/logrus v1.8.0
 	github.com/spacemonkeygo/monkit/v3 v3.0.22
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5
@@ -40,7 +41,6 @@ require (
 	github.com/cheggaaa/pb v1.0.29 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dchest/siphash v1.2.1 // indirect
-	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
 	github.com/djherbis/atime v1.0.0 // indirect
 	github.com/dswarbrick/smart v0.0.0-20190505152634-909a45200d6d // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
@@ -57,6 +57,8 @@ require (
 	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/go-sql-driver/mysql v1.5.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
+	github.com/golang/mock v1.6.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.3 // indirect
 	github.com/gomodule/redigo v1.8.3 // indirect
@@ -80,6 +82,7 @@ require (
 	github.com/klauspost/readahead v1.3.1 // indirect
 	github.com/klauspost/reedsolomon v1.9.11 // indirect
 	github.com/lib/pq v1.10.2 // indirect
+	github.com/magefile/mage v1.10.0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.6 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
@@ -170,3 +173,5 @@ require (
 	storj.io/monkit-jaeger v0.0.0-20240221095020-52b0792fa6cd // indirect
 	storj.io/picobuf v0.0.3 // indirect
 )
+
+replace storj.io/minio => github.com/deweb-services/storj v0.1.22

go.sum

@@ -114,6 +114,10 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dchest/siphash v1.2.1 h1:4cLinnzVJDKxTCl9B01807Yiy+W7ZzVHj/KIroQRvT4=
 github.com/dchest/siphash v1.2.1/go.mod h1:q+IRvb2gOSrUnYoPqHiyHXS0FOBBOdl6tONBlVnOnt4=
+github.com/deweb-services/go-nodeshift v0.0.2 h1:RfmgZT1C+siSVH59HqncaebY6PHntNuB8KYENByxOZ4=
+github.com/deweb-services/go-nodeshift v0.0.2/go.mod h1:CWeHG5NOBoaMrUeLMkmDj82ndI5OVpHVthtBVV+0cNE=
+github.com/deweb-services/storj v0.1.22 h1:ixl3vEWN1E90Dlp0Mmq5QE9LALIsyX5dDW2KfXRx5B0=
+github.com/deweb-services/storj v0.1.22/go.mod h1:4sIxKxTwwuIz9zFXMEwQJOdjW6Uj2gnweAv9Aw29idY=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/djherbis/atime v1.0.0 h1:ySLvBAM0EvOGaX7TI4dAM5lWj+RdJUCKtGSEHN8SGBg=
@@ -189,6 +193,8 @@ github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
+github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -197,6 +203,7 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
 github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
+github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -286,6 +293,8 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
@@ -369,6 +378,7 @@ github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0U
 github.com/lucas-clemente/quic-go v0.27.1/go.mod h1:AzgQoPda7N+3IqMMMkywBKggIFo2KT6pfnlrQ2QieeI=
 github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
+github.com/magefile/mage v1.10.0 h1:3HiXzCUY12kh9bIuyXShaVe529fJfyqoVM42o/uom2g=
 github.com/magefile/mage v1.10.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
 github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
 github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
@@ -620,6 +630,7 @@ github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5k
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/sirupsen/logrus v1.8.0 h1:nfhvjKcUMhBMVqbKHJlk5RPrrfYr/NMo3692g0dwfWU=
 github.com/sirupsen/logrus v1.8.0/go.mod h1:4GuYW9TZmE769R5STWrRakJc4UqQ3+QQ95fyz7ENv1A=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
@@ -1062,8 +1073,6 @@ storj.io/gateway v1.9.0 h1:SvejVcZLfyi7vLVP3uS6mXrTC4iY1Xq/dhfHMcXZfAs=
 storj.io/gateway v1.9.0/go.mod h1:pbwQ9F3vv2VNXUxievdbULlgQHduAidshvUN4utQOyo=
 storj.io/infectious v0.0.2 h1:rGIdDC/6gNYAStsxsZU79D/MqFjNyJc1tsyyj9sTl7Q=
 storj.io/infectious v0.0.2/go.mod h1:QEjKKww28Sjl1x8iDsjBpOM4r1Yp8RsowNcItsZJ1Vs=
-storj.io/minio v0.0.0-20230901173759-f1d4dd341feb h1:v8nZcUG8KU5GYPXdAtGUpT7e7aF5kbVqm+66NRizl5o=
-storj.io/minio v0.0.0-20230901173759-f1d4dd341feb/go.mod h1:tNaALPgx7e3WNMauIXVKw2ygXk4+o11XIR672akiI7I=
 storj.io/monkit-jaeger v0.0.0-20240221095020-52b0792fa6cd h1:XkLxAvogynNQu/1gM/F5MjBRgUGYdqWXWcwccZsm1SQ=
 storj.io/monkit-jaeger v0.0.0-20240221095020-52b0792fa6cd/go.mod h1:nwyyJiAFAPRu81qBfAE5cBGkVXROYSNJu9pGXLtStHE=
 storj.io/picobuf v0.0.3 h1:xAUPB5ZUGfxkqd3bnw3zp01kkWb9wlhg4vtZWUs2S9A=

miniogw/errors.go

@@ -0,0 +1,5 @@
+package miniogw
+
+import "errors"
+
+var ErrProjectNotDefined = errors.New("project is not defined")

miniogw/gateway.go

@@ -8,9 +8,11 @@
 	"context"
 	"encoding/hex"
 	"errors"
+	"fmt"
 	"io"
 	"net/http"
 	"sort"
+	"strconv"
 	"strings"
 	"syscall"
 	"time"
@@ -25,8 +27,7 @@
 	minio "storj.io/minio/cmd"
 	"storj.io/minio/cmd/config/storageclass"
 	xhttp "storj.io/minio/cmd/http"
-	"storj.io/minio/pkg/auth"
 	"storj.io/minio/pkg/bucket/versioning"
 	"storj.io/minio/pkg/hash"
 	"storj.io/minio/pkg/madmin"
 	"storj.io/private/version"
@@ -34,6 +35,8 @@
 	"storj.io/uplink/private/bucket"
 	uo "storj.io/uplink/private/object"
 	versioned "storj.io/uplink/private/object"
+
+	nodeshift "github.com/deweb-services/go-nodeshift/project"
 )
 
 var (
@@ -133,10 +136,17 @@
 }
 
 // NewGatewayLayer implements cmd.Gateway.
-func (gateway *Gateway) NewGatewayLayer(logger debugLogger, creds auth.Credentials) (minio.ObjectLayer, error) {
+func (gateway *Gateway) NewGatewayLayer(logger debugLogger, bucket string) (minio.ObjectLayer, error) {
+	w := NewWorker(bucket)
+
+	go w.Run()
+
 	return &gatewayLayer{
 		logger:              logger,
 		compatibilityConfig: gateway.compatibilityConfig,
+
+		internalBucket: bucket,
+		worker:         w,
 	}, nil
 }
 
@@ -147,14 +157,21 @@
 
 type gatewayLayer struct {
 	logger debugLogger
+	worker worker
 	minio.GatewayUnsupported
 	compatibilityConfig S3CompatibilityConfig
+	internalBucket      string
 }
 
-type debugLogger interface {
-	Info(args ...interface{})
-	Infof(format string, args ...interface{})
-}
+type (
+	debugLogger interface {
+		Info(args ...interface{})
+		Infof(format string, args ...interface{})
+	}
+	worker interface {
+		Add(ctx context.Context) error
+	}
+)
 
 // Shutdown is a no-op.
 func (layer *gatewayLayer) Shutdown(ctx context.Context) (err error) {
@@ -173,95 +190,77 @@
 func (layer *gatewayLayer) MakeBucketWithLocation(ctx context.Context, bucket string, opts minio.BucketOptions) (err error) {
 	defer mon.Task()(&ctx)(&err)
 
-	if err := ValidateBucket(ctx, bucket); err != nil {
-		return minio.BucketNameInvalid{Bucket: bucket}
-	}
-
-	project, err := projectFromContext(ctx, bucket, "")
-	if err != nil {
-		return err
-	}
-
-	_, err = project.CreateBucket(ctx, bucket)
+	_, err = layer.PutObject(ctx, layer.internalBucket, bucket, nil, minio.ObjectOptions{
+		UserDefined: map[string]string{"size": "0"},
+	})
 
 	return ConvertError(err, bucket, "")
 }
 
 func (layer *gatewayLayer) GetBucketInfo(ctx context.Context, bucketName string) (bucketInfo minio.BucketInfo, err error) {
 	defer mon.Task()(&ctx)(&err)
 
-	if err := ValidateBucket(ctx, bucketName); err != nil {
-		return minio.BucketInfo{}, minio.BucketNameInvalid{Bucket: bucketName}
-	}
-
-	project, err := projectFromContext(ctx, bucketName, "")
+	o, err := layer.GetObjectInfo(ctx, layer.internalBucket, bucketName+"/", minio.ObjectOptions{})
 	if err != nil {
-		return minio.BucketInfo{}, err
+		return minio.BucketInfo{}, fmt.Errorf("failed to get bucket info: %w", err)
 	}
 
-	bucket, err := project.StatBucket(ctx, bucketName)
-	if err != nil {
-		return minio.BucketInfo{}, ConvertError(err, bucketName, "")
+	size := int64(0)
+	if o.UserDefined["size"] != "" {
+		var err error
+		size, err = strconv.ParseInt(o.UserDefined["size"], 10, 64)
+		if err != nil {
+			return minio.BucketInfo{}, fmt.Errorf("failed to convert size: %w", err)
+		}
 	}
 
 	return minio.BucketInfo{
-		Name:    bucket.Name,
-		Created: bucket.Created,
+		Name:    o.Name,
+		Created: o.ModTime,
+		Size:    size,
 	}, nil
 }
 
-func (layer *gatewayLayer) ListBuckets(ctx context.Context) (items []minio.BucketInfo, err error) {
-	defer mon.Task()(&ctx)(&err)
+func (layer *gatewayLayer) ListBuckets(ctx context.Context) (_ []minio.BucketInfo, err error) {
+	projectUUID, ok := nodeshift.GetProjectUUID(ctx)
+	if !ok {
+		return nil, fmt.Errorf("failed to get project uuid: %w", ErrProjectNotDefined)
+	}
 
-	project, err := projectFromContext(ctx, "", "")
+	objects, err := layer.ListObjects(ctx, layer.internalBucket, projectUUID+"/", "", "/", 10000)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to list objects: %w", err)
 	}
 
-	buckets := project.ListBuckets(ctx, nil)
-	for buckets.Next() {
-		info := buckets.Item()
-		items = append(items, minio.BucketInfo{
-			Name:    info.Name,
-			Created: info.Created,
+	resp := make([]minio.BucketInfo, 0, len(objects.Objects))
+	for _, o := range objects.Objects {
+		size := int64(0)
+		if o.UserDefined["size"] != "" {
+			var err error
+			size, err = strconv.ParseInt(o.UserDefined["size"], 10, 64)
+			if err != nil {
+				return nil, fmt.Errorf("failed to convert size: %w", err)
+			}
+		}
+
+		resp = append(resp, minio.BucketInfo{
+			Name:    o.Name,
+			Created: o.ModTime,
+			Size:    size,
 		})
 	}
-	if buckets.Err() != nil {
-		return nil, ConvertError(buckets.Err(), "", "")
+
+	if err := layer.worker.Add(ctx); err != nil {
+		return nil, fmt.Errorf("failed to push data to worker: %w", err)
 	}
-	return items, nil
+
+	return resp, nil
 }
 
 func (layer *gatewayLayer) DeleteBucket(ctx context.Context, bucket string, forceDelete bool) (err error) {
 	defer mon.Task()(&ctx)(&err)
 
-	if err := ValidateBucket(ctx, bucket); err != nil {
-		return minio.BucketNameInvalid{Bucket: bucket}
-	}
-
-	project, err := projectFromContext(ctx, bucket, "")
-	if err != nil {
-		return err
-	}
-
-	if forceDelete {
-		_, err = project.DeleteBucketWithObjects(ctx, bucket)
-		return ConvertError(err, bucket, "")
-	}
-
-	_, err = project.DeleteBucket(ctx, bucket)
-	if errs.Is(err, uplink.ErrBucketNotEmpty) {
-		// Check if the bucket contains any non-pending objects. If it doesn't,
-		// this would mean there were initiated non-committed and non-aborted
-		// multipart uploads. Other S3 implementations allow deletion of such
-		// buckets, but with uplink, we need to explicitly force bucket
-		// deletion.
-		it := project.ListObjects(ctx, bucket, nil)
-		if !it.Next() && it.Err() == nil {
-			_, err = project.DeleteBucketWithObjects(ctx, bucket)
-			return ConvertError(err, bucket, "")
-		}
-	}
+	_, err = layer.DeleteObject(ctx, layer.internalBucket, bucket, minio.ObjectOptions{})
 
 	return ConvertError(err, bucket, "")
 }