From 000e097a21ff802e13d49db33e4fce9d16a468b6 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 6 Oct 2026 10:47:27 +0100 Subject: [PATCH 1/5] feat: add Blake3 object attribute extension --- s3api/controllers/object-get_test.go | 31 ++++++++++++++++++++++++++++ s3api/utils/utils.go | 3 +++ s3api/utils/utils_test.go | 28 +++++++++++++++++++++++++ s3response/blake3_test.go | 26 +++++++++++++++++++++++ s3response/s3response.go | 21 ++++++++++++++++++- 5 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 s3response/blake3_test.go diff --git a/s3api/controllers/object-get_test.go b/s3api/controllers/object-get_test.go index 8674bc8c4..1a9ffa3ce 100644 --- a/s3api/controllers/object-get_test.go +++ b/s3api/controllers/object-get_test.go @@ -607,6 +607,37 @@ func TestS3ApiController_GetObjectAttributes(t *testing.T) { err: s3err.GetAPIError(s3err.ErrNoSuchBucket), }, }, + { + name: "Blake3 attribute accepted", + input: testInput{ + locals: defaultLocals, + beRes: s3response.GetObjectAttributesResponse{ + DeleteMarker: &delMarker, + LastModified: &lastModTime, + VersionId: utils.GetStringPtr("versionId"), + ETag: &etag, + Blake3: &s3response.Blake3Tree{CID: "bafkr4i", Group: 22, Leaves: []string{"AA=="}}, + }, + headers: map[string]string{ + "X-Amz-Object-Attributes": "Blake3", + }, + }, + output: testOutput{ + response: &Response{ + Headers: map[string]*string{ + "x-amz-version-id": utils.GetStringPtr("versionId"), + "x-amz-delete-marker": utils.GetStringPtr("true"), + "Last-Modified": &timeFormatted, + }, + Data: s3response.GetObjectAttributesResponse{ + Blake3: &s3response.Blake3Tree{CID: "bafkr4i", Group: 22, Leaves: []string{"AA=="}}, + }, + MetaOpts: &MetaOptions{ + BucketOwner: "root", + }, + }, + }, + }, { name: "successful response", input: testInput{ diff --git a/s3api/utils/utils.go b/s3api/utils/utils.go index 4722c29e0..54009bd15 100644 --- a/s3api/utils/utils.go +++ b/s3api/utils/utils.go @@ -446,6 +446,9 @@ func FilterObjectAttributes(attrs map[s3response.ObjectAttributes]struct{}, outp if _, ok := attrs[s3response.ObjectAttributesChecksum]; !ok { output.Checksum = nil } + if _, ok := attrs[s3response.ObjectAttributesBlake3]; !ok { + output.Blake3 = nil + } return output } diff --git a/s3api/utils/utils_test.go b/s3api/utils/utils_test.go index ebac61fae..8a63bc9de 100644 --- a/s3api/utils/utils_test.go +++ b/s3api/utils/utils_test.go @@ -610,6 +610,34 @@ func TestFilterObjectAttributes(t *testing.T) { }, want: s3response.GetObjectAttributesResponse{ETag: &etag}, }, + { + name: "drop Blake3 when not requested", + args: args{ + attrs: map[s3response.ObjectAttributes]struct{}{ + s3response.ObjectAttributesEtag: {}, + }, + output: s3response.GetObjectAttributesResponse{ + ETag: &etag, + Blake3: &s3response.Blake3Tree{CID: "bafkr4i", Group: 22, Leaves: []string{"AA=="}}, + }, + }, + want: s3response.GetObjectAttributesResponse{ETag: &etag}, + }, + { + name: "keep Blake3 when requested", + args: args{ + attrs: map[s3response.ObjectAttributes]struct{}{ + s3response.ObjectAttributesBlake3: {}, + }, + output: s3response.GetObjectAttributesResponse{ + ETag: &etag, + Blake3: &s3response.Blake3Tree{CID: "bafkr4i", Group: 22, Leaves: []string{"AA=="}}, + }, + }, + want: s3response.GetObjectAttributesResponse{ + Blake3: &s3response.Blake3Tree{CID: "bafkr4i", Group: 22, Leaves: []string{"AA=="}}, + }, + }, { name: "keep multiple props", args: args{ diff --git a/s3response/blake3_test.go b/s3response/blake3_test.go new file mode 100644 index 000000000..111c66517 --- /dev/null +++ b/s3response/blake3_test.go @@ -0,0 +1,26 @@ +package s3response + +import ( + "encoding/xml" + "strings" + "testing" +) + +// TestBlake3TreeXML pins the element shape the Blake3 object attribute +// documents: CID, Group, and one Leaf per block under Leaves. +func TestBlake3TreeXML(t *testing.T) { + res := GetObjectAttributesResponse{ + Blake3: &Blake3Tree{CID: "bafkr4iexample", Group: 22, Leaves: []string{"AAEC", "AwQF"}}, + } + out, err := xml.Marshal(res) + if err != nil { + t.Fatal(err) + } + want := "bafkr4iexample22AAECAwQF" + if !strings.Contains(string(out), want) { + t.Fatalf("marshaled %s, want it to contain %s", out, want) + } + if out, _ := xml.Marshal(GetObjectAttributesResponse{}); strings.Contains(string(out), "Blake3") { + t.Fatalf("a nil tree must omit the element: %s", out) + } +} diff --git a/s3response/s3response.go b/s3response/s3response.go index 3d4d66420..f752c1868 100644 --- a/s3response/s3response.go +++ b/s3response/s3response.go @@ -111,6 +111,10 @@ const ( ObjectAttributesObjectParts ObjectAttributes = "ObjectParts" ObjectAttributesStorageClass ObjectAttributes = "StorageClass" ObjectAttributesObjectSize ObjectAttributes = "ObjectSize" + // ObjectAttributesBlake3 is an extension to the S3 API: the object's + // BLAKE3 digest and the tree a client verifies ranged reads against + // (see Blake3Tree). AWS rejects the name with InvalidArgument. + ObjectAttributesBlake3 ObjectAttributes = "Blake3" ) func (o ObjectAttributes) IsValid() bool { @@ -118,7 +122,8 @@ func (o ObjectAttributes) IsValid() bool { o == ObjectAttributesEtag || o == ObjectAttributesObjectParts || o == ObjectAttributesObjectSize || - o == ObjectAttributesStorageClass + o == ObjectAttributesStorageClass || + o == ObjectAttributesBlake3 } type GetObjectAttributesResponse struct { @@ -127,6 +132,7 @@ type GetObjectAttributesResponse struct { StorageClass types.StorageClass `xml:",omitempty"` ObjectParts *ObjectParts Checksum *types.Checksum + Blake3 *Blake3Tree `xml:"Blake3,omitempty"` // Not included in the response body VersionId *string @@ -134,6 +140,19 @@ type GetObjectAttributesResponse struct { DeleteMarker *bool } +// Blake3Tree is the Blake3 object attribute: the object's BLAKE3 digest as a +// CID (version 1, raw codec, blake3 multihash) and the chaining values of its +// group-aligned blocks, from which a client verifies a ranged read. Group is +// the block size as a base-2 exponent of bytes; Leaves holds one 32-byte +// chaining value per block in order, base64-encoded like the checksums in +// ObjectParts. A backend that records no tree for an object leaves the field +// nil and the element is omitted. +type Blake3Tree struct { + CID string `xml:"CID"` + Group uint8 `xml:"Group"` + Leaves []string `xml:"Leaves>Leaf"` +} + type ObjectParts struct { // PartsCount is the object's total multipart part count (AWS , // the SDK's TotalPartsCount). Emitted for every completed multipart object. From 2a70102c1bca52dfb60694fff9b29fe68d0bc813 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 6 Oct 2026 11:03:58 +0100 Subject: [PATCH 2/5] refactor: encode outboard in standard format --- s3api/controllers/object-get_test.go | 4 ++-- s3api/utils/utils_test.go | 6 +++--- s3response/blake3_test.go | 6 +++--- s3response/s3response.go | 19 +++++++++++-------- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/s3api/controllers/object-get_test.go b/s3api/controllers/object-get_test.go index 1a9ffa3ce..a85182b11 100644 --- a/s3api/controllers/object-get_test.go +++ b/s3api/controllers/object-get_test.go @@ -616,7 +616,7 @@ func TestS3ApiController_GetObjectAttributes(t *testing.T) { LastModified: &lastModTime, VersionId: utils.GetStringPtr("versionId"), ETag: &etag, - Blake3: &s3response.Blake3Tree{CID: "bafkr4i", Group: 22, Leaves: []string{"AA=="}}, + Blake3: &s3response.Blake3Tree{CID: "bafkr4i", Group: 22, Outboard: "AAAAAAAAAAA="}, }, headers: map[string]string{ "X-Amz-Object-Attributes": "Blake3", @@ -630,7 +630,7 @@ func TestS3ApiController_GetObjectAttributes(t *testing.T) { "Last-Modified": &timeFormatted, }, Data: s3response.GetObjectAttributesResponse{ - Blake3: &s3response.Blake3Tree{CID: "bafkr4i", Group: 22, Leaves: []string{"AA=="}}, + Blake3: &s3response.Blake3Tree{CID: "bafkr4i", Group: 22, Outboard: "AAAAAAAAAAA="}, }, MetaOpts: &MetaOptions{ BucketOwner: "root", diff --git a/s3api/utils/utils_test.go b/s3api/utils/utils_test.go index 8a63bc9de..55d897524 100644 --- a/s3api/utils/utils_test.go +++ b/s3api/utils/utils_test.go @@ -618,7 +618,7 @@ func TestFilterObjectAttributes(t *testing.T) { }, output: s3response.GetObjectAttributesResponse{ ETag: &etag, - Blake3: &s3response.Blake3Tree{CID: "bafkr4i", Group: 22, Leaves: []string{"AA=="}}, + Blake3: &s3response.Blake3Tree{CID: "bafkr4i", Group: 22, Outboard: "AAAAAAAAAAA="}, }, }, want: s3response.GetObjectAttributesResponse{ETag: &etag}, @@ -631,11 +631,11 @@ func TestFilterObjectAttributes(t *testing.T) { }, output: s3response.GetObjectAttributesResponse{ ETag: &etag, - Blake3: &s3response.Blake3Tree{CID: "bafkr4i", Group: 22, Leaves: []string{"AA=="}}, + Blake3: &s3response.Blake3Tree{CID: "bafkr4i", Group: 22, Outboard: "AAAAAAAAAAA="}, }, }, want: s3response.GetObjectAttributesResponse{ - Blake3: &s3response.Blake3Tree{CID: "bafkr4i", Group: 22, Leaves: []string{"AA=="}}, + Blake3: &s3response.Blake3Tree{CID: "bafkr4i", Group: 22, Outboard: "AAAAAAAAAAA="}, }, }, { diff --git a/s3response/blake3_test.go b/s3response/blake3_test.go index 111c66517..56cfdffbc 100644 --- a/s3response/blake3_test.go +++ b/s3response/blake3_test.go @@ -7,16 +7,16 @@ import ( ) // TestBlake3TreeXML pins the element shape the Blake3 object attribute -// documents: CID, Group, and one Leaf per block under Leaves. +// documents: CID, Group and the base64 Outboard. func TestBlake3TreeXML(t *testing.T) { res := GetObjectAttributesResponse{ - Blake3: &Blake3Tree{CID: "bafkr4iexample", Group: 22, Leaves: []string{"AAEC", "AwQF"}}, + Blake3: &Blake3Tree{CID: "bafkr4iexample", Group: 22, Outboard: "AAAAAAAAAAA="}, } out, err := xml.Marshal(res) if err != nil { t.Fatal(err) } - want := "bafkr4iexample22AAECAwQF" + want := "bafkr4iexample22AAAAAAAAAAA=" if !strings.Contains(string(out), want) { t.Fatalf("marshaled %s, want it to contain %s", out, want) } diff --git a/s3response/s3response.go b/s3response/s3response.go index f752c1868..39ab1dea8 100644 --- a/s3response/s3response.go +++ b/s3response/s3response.go @@ -141,16 +141,19 @@ type GetObjectAttributesResponse struct { } // Blake3Tree is the Blake3 object attribute: the object's BLAKE3 digest as a -// CID (version 1, raw codec, blake3 multihash) and the chaining values of its -// group-aligned blocks, from which a client verifies a ranged read. Group is -// the block size as a base-2 exponent of bytes; Leaves holds one 32-byte -// chaining value per block in order, base64-encoded like the checksums in -// ObjectParts. A backend that records no tree for an object leaves the field +// CID (version 1, raw codec, blake3 multihash) and the Bao outboard a client +// verifies ranged reads with. Group is the Bao block size as a base-2 +// exponent of bytes (a Bao library's chunk log is Group minus 10). Outboard +// is the standard pre-order Bao outboard, base64-encoded: the object size as +// 8 little-endian bytes, then the chaining-value pair of each parent node +// above the block size, root first. A Bao library loads the CID's digest, +// the block size and the outboard and verifies any block-aligned range of +// the object. A backend that records no tree for an object leaves the field // nil and the element is omitted. type Blake3Tree struct { - CID string `xml:"CID"` - Group uint8 `xml:"Group"` - Leaves []string `xml:"Leaves>Leaf"` + CID string `xml:"CID"` + Group uint8 `xml:"Group"` + Outboard string `xml:"Outboard"` } type ObjectParts struct { From a094a7f5d783793bdeffe5457310db9389ccb82a Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 6 Oct 2026 11:05:31 +0100 Subject: [PATCH 3/5] fix: GO-2026-6503 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9310503ab..b7ceab3a0 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/nats-io/nats.go v1.52.0 github.com/oklog/ulid/v2 v2.1.1 github.com/pkg/xattr v0.4.12 - github.com/rabbitmq/amqp091-go v1.12.0 + github.com/rabbitmq/amqp091-go v1.13.0 github.com/segmentio/kafka-go v0.4.51 github.com/smira/go-statsd v1.3.4 github.com/stretchr/testify v1.11.1 diff --git a/go.sum b/go.sum index a0798933e..e5685d036 100644 --- a/go.sum +++ b/go.sum @@ -157,8 +157,8 @@ github.com/pkg/xattr v0.4.12 h1:rRTkSyFNTRElv6pkA3zpjHpQ90p/OdHQC1GmGh1aTjM= github.com/pkg/xattr v0.4.12/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rabbitmq/amqp091-go v1.12.0 h1:V0v14Iqfs+MwHWihJt/nGS5Ulu0vw572b2Co3mwunkI= -github.com/rabbitmq/amqp091-go v1.12.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/rabbitmq/amqp091-go v1.13.0 h1:L8NA1WtF76C6KA3LAoufjfLgbist/If1UQYcsOjtxXA= +github.com/rabbitmq/amqp091-go v1.13.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= From 5824567a679446b65062b6b792e2979b5c18a1ff Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 6 Oct 2026 13:23:50 +0100 Subject: [PATCH 4/5] fix: add requested attributes to GetObjectAttributesInput --- s3api/controllers/object-get.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/s3api/controllers/object-get.go b/s3api/controllers/object-get.go index dc35a21ab..1f3061249 100644 --- a/s3api/controllers/object-get.go +++ b/s3api/controllers/object-get.go @@ -341,6 +341,13 @@ func (c S3ApiController) GetObjectAttributes(ctx fiber.Ctx) (*Response, error) { }, err } + // The requested attributes go to the backend too, so one that is costly + // to produce (the Blake3 outboard) is built only when asked for; the + // filter below still decides what the response carries. + requested := make([]types.ObjectAttributes, 0, len(attrs)) + for a := range attrs { + requested = append(requested, types.ObjectAttributes(a)) + } res, err := c.be.GetObjectAttributes(ctx.RequestCtx(), &s3.GetObjectAttributesInput{ Bucket: &bucket, @@ -348,6 +355,7 @@ func (c S3ApiController) GetObjectAttributes(ctx fiber.Ctx) (*Response, error) { PartNumberMarker: &partNumberMarker, MaxParts: maxParts, VersionId: &versionId, + ObjectAttributes: requested, }) if err != nil { headers := map[string]*string{ From d43133bfde6281d218d7887d47d2acb3c04199c4 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 6 Oct 2026 15:03:57 +0100 Subject: [PATCH 5/5] refactor: rename group to chunk log --- s3api/controllers/object-get_test.go | 4 ++-- s3api/utils/utils_test.go | 6 +++--- s3response/blake3_test.go | 6 +++--- s3response/s3response.go | 19 ++++++++++--------- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/s3api/controllers/object-get_test.go b/s3api/controllers/object-get_test.go index a85182b11..5c998f52c 100644 --- a/s3api/controllers/object-get_test.go +++ b/s3api/controllers/object-get_test.go @@ -616,7 +616,7 @@ func TestS3ApiController_GetObjectAttributes(t *testing.T) { LastModified: &lastModTime, VersionId: utils.GetStringPtr("versionId"), ETag: &etag, - Blake3: &s3response.Blake3Tree{CID: "bafkr4i", Group: 22, Outboard: "AAAAAAAAAAA="}, + Blake3: &s3response.Blake3Tree{CID: "bafkr4i", ChunkLog: 12, Outboard: "AAAAAAAAAAA="}, }, headers: map[string]string{ "X-Amz-Object-Attributes": "Blake3", @@ -630,7 +630,7 @@ func TestS3ApiController_GetObjectAttributes(t *testing.T) { "Last-Modified": &timeFormatted, }, Data: s3response.GetObjectAttributesResponse{ - Blake3: &s3response.Blake3Tree{CID: "bafkr4i", Group: 22, Outboard: "AAAAAAAAAAA="}, + Blake3: &s3response.Blake3Tree{CID: "bafkr4i", ChunkLog: 12, Outboard: "AAAAAAAAAAA="}, }, MetaOpts: &MetaOptions{ BucketOwner: "root", diff --git a/s3api/utils/utils_test.go b/s3api/utils/utils_test.go index 55d897524..c3e05b653 100644 --- a/s3api/utils/utils_test.go +++ b/s3api/utils/utils_test.go @@ -618,7 +618,7 @@ func TestFilterObjectAttributes(t *testing.T) { }, output: s3response.GetObjectAttributesResponse{ ETag: &etag, - Blake3: &s3response.Blake3Tree{CID: "bafkr4i", Group: 22, Outboard: "AAAAAAAAAAA="}, + Blake3: &s3response.Blake3Tree{CID: "bafkr4i", ChunkLog: 12, Outboard: "AAAAAAAAAAA="}, }, }, want: s3response.GetObjectAttributesResponse{ETag: &etag}, @@ -631,11 +631,11 @@ func TestFilterObjectAttributes(t *testing.T) { }, output: s3response.GetObjectAttributesResponse{ ETag: &etag, - Blake3: &s3response.Blake3Tree{CID: "bafkr4i", Group: 22, Outboard: "AAAAAAAAAAA="}, + Blake3: &s3response.Blake3Tree{CID: "bafkr4i", ChunkLog: 12, Outboard: "AAAAAAAAAAA="}, }, }, want: s3response.GetObjectAttributesResponse{ - Blake3: &s3response.Blake3Tree{CID: "bafkr4i", Group: 22, Outboard: "AAAAAAAAAAA="}, + Blake3: &s3response.Blake3Tree{CID: "bafkr4i", ChunkLog: 12, Outboard: "AAAAAAAAAAA="}, }, }, { diff --git a/s3response/blake3_test.go b/s3response/blake3_test.go index 56cfdffbc..69f89506d 100644 --- a/s3response/blake3_test.go +++ b/s3response/blake3_test.go @@ -7,16 +7,16 @@ import ( ) // TestBlake3TreeXML pins the element shape the Blake3 object attribute -// documents: CID, Group and the base64 Outboard. +// documents: CID, ChunkLog and the base64 Outboard. func TestBlake3TreeXML(t *testing.T) { res := GetObjectAttributesResponse{ - Blake3: &Blake3Tree{CID: "bafkr4iexample", Group: 22, Outboard: "AAAAAAAAAAA="}, + Blake3: &Blake3Tree{CID: "bafkr4iexample", ChunkLog: 12, Outboard: "AAAAAAAAAAA="}, } out, err := xml.Marshal(res) if err != nil { t.Fatal(err) } - want := "bafkr4iexample22AAAAAAAAAAA=" + want := "bafkr4iexample12AAAAAAAAAAA=" if !strings.Contains(string(out), want) { t.Fatalf("marshaled %s, want it to contain %s", out, want) } diff --git a/s3response/s3response.go b/s3response/s3response.go index 39ab1dea8..a94ad865b 100644 --- a/s3response/s3response.go +++ b/s3response/s3response.go @@ -142,17 +142,18 @@ type GetObjectAttributesResponse struct { // Blake3Tree is the Blake3 object attribute: the object's BLAKE3 digest as a // CID (version 1, raw codec, blake3 multihash) and the Bao outboard a client -// verifies ranged reads with. Group is the Bao block size as a base-2 -// exponent of bytes (a Bao library's chunk log is Group minus 10). Outboard -// is the standard pre-order Bao outboard, base64-encoded: the object size as -// 8 little-endian bytes, then the chaining-value pair of each parent node -// above the block size, root first. A Bao library loads the CID's digest, -// the block size and the outboard and verifies any block-aligned range of -// the object. A backend that records no tree for an object leaves the field -// nil and the element is omitted. +// verifies ranged reads with. ChunkLog is the Bao block size as a base-2 +// exponent of 1 KiB BLAKE3 chunks, the value Bao libraries take as is +// (iroh's fixed block is 4). Outboard is the standard pre-order Bao +// outboard, base64-encoded: the object size as 8 little-endian bytes, then +// the chaining-value pair of each parent node above the block size, root +// first. A Bao library loads the CID's digest, the chunk log and the +// outboard and verifies any block-aligned range of the object. A backend +// that records no tree for an object leaves the field nil and the element +// is omitted. type Blake3Tree struct { CID string `xml:"CID"` - Group uint8 `xml:"Group"` + ChunkLog uint8 `xml:"ChunkLog"` Outboard string `xml:"Outboard"` }