feat: add Blake3 object attribute extension

go.mod

@@ -24,7 +24,7 @@ require (
 	github.com/nats-io/nats.go v1.52.0
 	github.com/oklog/ulid/v2 v2.1.1
 	github.com/pkg/xattr v0.4.12
-	github.com/rabbitmq/amqp091-go v1.12.0
+	github.com/rabbitmq/amqp091-go v1.13.0
 	github.com/segmentio/kafka-go v0.4.51
 	github.com/smira/go-statsd v1.3.4
 	github.com/stretchr/testify v1.11.1

go.sum

@@ -157,8 +157,8 @@ github.com/pkg/xattr v0.4.12 h1:rRTkSyFNTRElv6pkA3zpjHpQ90p/OdHQC1GmGh1aTjM=
 github.com/pkg/xattr v0.4.12/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rabbitmq/amqp091-go v1.12.0 h1:V0v14Iqfs+MwHWihJt/nGS5Ulu0vw572b2Co3mwunkI=
-github.com/rabbitmq/amqp091-go v1.12.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
+github.com/rabbitmq/amqp091-go v1.13.0 h1:L8NA1WtF76C6KA3LAoufjfLgbist/If1UQYcsOjtxXA=
+github.com/rabbitmq/amqp091-go v1.13.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
 github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
 github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=

s3api/controllers/object-get.go

@@ -341,13 +341,21 @@ func (c S3ApiController) GetObjectAttributes(ctx fiber.Ctx) (*Response, error) {
 		}, err
 	}
 
+	// The requested attributes go to the backend too, so one that is costly
+	// to produce (the Blake3 outboard) is built only when asked for; the
+	// filter below still decides what the response carries.
+	requested := make([]types.ObjectAttributes, 0, len(attrs))
+	for a := range attrs {
+		requested = append(requested, types.ObjectAttributes(a))
+	}
 	res, err := c.be.GetObjectAttributes(ctx.RequestCtx(),
 		&s3.GetObjectAttributesInput{
 			Bucket:           &bucket,
 			Key:              &key,
 			PartNumberMarker: &partNumberMarker,
 			MaxParts:         maxParts,
 			VersionId:        &versionId,
+			ObjectAttributes: requested,
 		})
 	if err != nil {
 		headers := map[string]*string{

s3api/controllers/object-get_test.go

@@ -607,6 +607,37 @@ func TestS3ApiController_GetObjectAttributes(t *testing.T) {
 				err: s3err.GetAPIError(s3err.ErrNoSuchBucket),
 			},
 		},
+		{
+			name: "Blake3 attribute accepted",
+			input: testInput{
+				locals: defaultLocals,
+				beRes: s3response.GetObjectAttributesResponse{
+					DeleteMarker: &delMarker,
+					LastModified: &lastModTime,
+					VersionId:    utils.GetStringPtr("versionId"),
+					ETag:         &etag,
+					Blake3:       &s3response.Blake3Tree{CID: "bafkr4i", ChunkLog: 12, Outboard: "AAAAAAAAAAA="},
+				},
+				headers: map[string]string{
+					"X-Amz-Object-Attributes": "Blake3",
+				},
+			},
+			output: testOutput{
+				response: &Response{
+					Headers: map[string]*string{
+						"x-amz-version-id":    utils.GetStringPtr("versionId"),
+						"x-amz-delete-marker": utils.GetStringPtr("true"),
+						"Last-Modified":       &timeFormatted,
+					},
+					Data: s3response.GetObjectAttributesResponse{
+						Blake3: &s3response.Blake3Tree{CID: "bafkr4i", ChunkLog: 12, Outboard: "AAAAAAAAAAA="},
+					},
+					MetaOpts: &MetaOptions{
+						BucketOwner: "root",
+					},
+				},
+			},
+		},
 		{
 			name: "successful response",
 			input: testInput{

s3api/utils/utils.go

@@ -446,6 +446,9 @@ func FilterObjectAttributes(attrs map[s3response.ObjectAttributes]struct{}, outp
 	if _, ok := attrs[s3response.ObjectAttributesChecksum]; !ok {
 		output.Checksum = nil
 	}
+	if _, ok := attrs[s3response.ObjectAttributesBlake3]; !ok {
+		output.Blake3 = nil
+	}
 
 	return output
 }

s3api/utils/utils_test.go

@@ -610,6 +610,34 @@ func TestFilterObjectAttributes(t *testing.T) {
 			},
 			want: s3response.GetObjectAttributesResponse{ETag: &etag},
 		},
+		{
+			name: "drop Blake3 when not requested",
+			args: args{
+				attrs: map[s3response.ObjectAttributes]struct{}{
+					s3response.ObjectAttributesEtag: {},
+				},
+				output: s3response.GetObjectAttributesResponse{
+					ETag:   &etag,
+					Blake3: &s3response.Blake3Tree{CID: "bafkr4i", ChunkLog: 12, Outboard: "AAAAAAAAAAA="},
+				},
+			},
+			want: s3response.GetObjectAttributesResponse{ETag: &etag},
+		},
+		{
+			name: "keep Blake3 when requested",
+			args: args{
+				attrs: map[s3response.ObjectAttributes]struct{}{
+					s3response.ObjectAttributesBlake3: {},
+				},
+				output: s3response.GetObjectAttributesResponse{
+					ETag:   &etag,
+					Blake3: &s3response.Blake3Tree{CID: "bafkr4i", ChunkLog: 12, Outboard: "AAAAAAAAAAA="},
+				},
+			},
+			want: s3response.GetObjectAttributesResponse{
+				Blake3: &s3response.Blake3Tree{CID: "bafkr4i", ChunkLog: 12, Outboard: "AAAAAAAAAAA="},
+			},
+		},
 		{
 			name: "keep multiple props",
 			args: args{

s3response/blake3_test.go

@@ -0,0 +1,26 @@
+package s3response
+
+import (
+	"encoding/xml"
+	"strings"
+	"testing"
+)
+
+// TestBlake3TreeXML pins the element shape the Blake3 object attribute
+// documents: CID, ChunkLog and the base64 Outboard.
+func TestBlake3TreeXML(t *testing.T) {
+	res := GetObjectAttributesResponse{
+		Blake3: &Blake3Tree{CID: "bafkr4iexample", ChunkLog: 12, Outboard: "AAAAAAAAAAA="},
+	}
+	out, err := xml.Marshal(res)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := "<Blake3><CID>bafkr4iexample</CID><ChunkLog>12</ChunkLog><Outboard>AAAAAAAAAAA=</Outboard></Blake3>"
+	if !strings.Contains(string(out), want) {
+		t.Fatalf("marshaled %s, want it to contain %s", out, want)
+	}
+	if out, _ := xml.Marshal(GetObjectAttributesResponse{}); strings.Contains(string(out), "Blake3") {
+		t.Fatalf("a nil tree must omit the element: %s", out)
+	}
+}

s3response/s3response.go

@@ -111,14 +111,19 @@ const (
 	ObjectAttributesObjectParts  ObjectAttributes = "ObjectParts"
 	ObjectAttributesStorageClass ObjectAttributes = "StorageClass"
 	ObjectAttributesObjectSize   ObjectAttributes = "ObjectSize"
+	// ObjectAttributesBlake3 is an extension to the S3 API: the object's
+	// BLAKE3 digest and the tree a client verifies ranged reads against
+	// (see Blake3Tree). AWS rejects the name with InvalidArgument.
+	ObjectAttributesBlake3 ObjectAttributes = "Blake3"
 )
 
 func (o ObjectAttributes) IsValid() bool {
 	return o == ObjectAttributesChecksum ||
 		o == ObjectAttributesEtag ||
 		o == ObjectAttributesObjectParts ||
 		o == ObjectAttributesObjectSize ||
-		o == ObjectAttributesStorageClass
+		o == ObjectAttributesStorageClass ||
+		o == ObjectAttributesBlake3
 }
 
 type GetObjectAttributesResponse struct {
@@ -127,13 +132,31 @@ type GetObjectAttributesResponse struct {
 	StorageClass types.StorageClass `xml:",omitempty"`
 	ObjectParts  *ObjectParts
 	Checksum     *types.Checksum
+	Blake3       *Blake3Tree `xml:"Blake3,omitempty"`
 
 	// Not included in the response body
 	VersionId    *string
 	LastModified *time.Time
 	DeleteMarker *bool
 }
 
+// Blake3Tree is the Blake3 object attribute: the object's BLAKE3 digest as a
+// CID (version 1, raw codec, blake3 multihash) and the Bao outboard a client
+// verifies ranged reads with. ChunkLog is the Bao block size as a base-2
+// exponent of 1 KiB BLAKE3 chunks, the value Bao libraries take as is
+// (iroh's fixed block is 4). Outboard is the standard pre-order Bao
+// outboard, base64-encoded: the object size as 8 little-endian bytes, then
+// the chaining-value pair of each parent node above the block size, root
+// first. A Bao library loads the CID's digest, the chunk log and the
+// outboard and verifies any block-aligned range of the object. A backend
+// that records no tree for an object leaves the field nil and the element
+// is omitted.
+type Blake3Tree struct {
+	CID      string `xml:"CID"`
+	ChunkLog uint8  `xml:"ChunkLog"`
+	Outboard string `xml:"Outboard"`
+}
+
 type ObjectParts struct {
 	// PartsCount is the object's total multipart part count (AWS <PartsCount>,
 	// the SDK's TotalPartsCount). Emitted for every completed multipart object.