diff --git a/commands/blob/abort.go b/commands/blob/abort.go index c6b4ca8..d6d7ef3 100644 --- a/commands/blob/abort.go +++ b/commands/blob/abort.go @@ -19,9 +19,9 @@ type AbortOK = commands.Unit // // Served by the upload service (subject = the space). A parked blob has no // registration or acceptance to look the storage node up by, so the service -// recovers it from the Cause receipt chain and forwards a `/blob/reject` -// (Cause itself is not forwarded — it is routing metadata, meaningless to -// the node). A missing or unknown Cause fails with MissingCause. Blobs the +// recovers it from the receipt chain of the Add task, and forwards a +// `/blob/reject` of the allocation the add made. A missing or unknown Add +// fails with MissingCause. Blobs the // space has accepted are released via `/blob/remove` instead; if the node // refuses the translated reject with BlobAccepted, the service surfaces that // named failure in the abort receipt. The abort mutates no upload-service @@ -32,9 +32,8 @@ type AbortOK = commands.Unit var Abort = binding.Bind[*AbortArguments, *AbortOK](command.MustParse("/blob/abort")) // MissingCauseErrorName is the stable receipt-failure name when an abort's -// Cause is missing or does not resolve to a known `/blob/add` task — -// without it the upload service cannot recover which storage node holds the -// parked blob. +// Add is missing or does not resolve to a known `/blob/add` task — without it +// the upload service cannot recover which storage node holds the parked blob. const MissingCauseErrorName = "MissingCause" -var ErrMissingCause = errors.New(MissingCauseErrorName, "abort requires the cause of the /blob/add task that parked the blob") +var ErrMissingCause = errors.New(MissingCauseErrorName, "abort requires the /blob/add task that parked the blob") diff --git a/commands/blob/abort_test.go b/commands/blob/abort_test.go index 19ac4c6..0667dd5 100644 --- a/commands/blob/abort_test.go +++ b/commands/blob/abort_test.go @@ -13,16 +13,12 @@ import ( // Round-trips AbortArguments through cbor. func TestAbortArgumentsRoundTrip(t *testing.T) { - in := blob.AbortArguments{ - Digest: testutil.RandomMultihash(t), - Cause: testutil.RandomCID(t), - } + in := blob.AbortArguments{Add: testutil.RandomCID(t)} var buf bytes.Buffer require.NoError(t, in.MarshalCBOR(&buf)) var out blob.AbortArguments require.NoError(t, out.UnmarshalCBOR(&buf)) - require.Equal(t, in.Digest, out.Digest) - require.Equal(t, in.Cause, out.Cause) + require.Equal(t, in, out) } // Round-trips ReleaseArguments through cbor. @@ -41,16 +37,18 @@ func TestReleaseArgumentsRoundTrip(t *testing.T) { require.Equal(t, in.Cause, out.Cause) } -// Round-trips RejectArguments through cbor. +// Round-trips RejectArguments through cbor and dag-json. func TestRejectArgumentsRoundTrip(t *testing.T) { - in := blob.RejectArguments{ - Space: testutil.RandomDID(t), - Digest: testutil.RandomMultihash(t), - } + in := blob.RejectArguments{Allocation: testutil.RandomCID(t)} var buf bytes.Buffer require.NoError(t, in.MarshalCBOR(&buf)) var out blob.RejectArguments require.NoError(t, out.UnmarshalCBOR(&buf)) - require.Equal(t, in.Space, out.Space) - require.Equal(t, in.Digest, out.Digest) + require.Equal(t, in, out) + + var js bytes.Buffer + require.NoError(t, in.MarshalDagJSON(&js)) + var outJS blob.RejectArguments + require.NoError(t, outJS.UnmarshalDagJSON(&js)) + require.Equal(t, in, outJS) } diff --git a/commands/blob/accept.go b/commands/blob/accept.go index 9494302..62ccd54 100644 --- a/commands/blob/accept.go +++ b/commands/blob/accept.go @@ -4,7 +4,17 @@ package blob import ( "github.com/fil-forge/ucantone/binding" + "github.com/fil-forge/ucantone/errors" "github.com/fil-forge/ucantone/ucan/command" ) var Accept = binding.Bind[*AcceptArguments, *AcceptOK](command.MustParse("/blob/accept")) + +// BlobDigestMismatchErrorName is the stable receipt-failure name when the +// digest the storage node computed for the received data differs from the +// digest in the accept arguments or, when they name only a digest code, the +// digest in the `/http/put` receipt. +const BlobDigestMismatchErrorName = "BlobDigestMismatch" + +// ErrBlobDigestMismatch is the failure for a digest mismatch. +var ErrBlobDigestMismatch = errors.New(BlobDigestMismatchErrorName, "received data does not hash to the reported digest") diff --git a/commands/blob/cbor_gen.go b/commands/blob/cbor_gen.go index b870181..ba008e3 100644 --- a/commands/blob/cbor_gen.go +++ b/commands/blob/cbor_gen.go @@ -33,7 +33,7 @@ func (t *AllocateArguments) MarshalCBOR(w io.Writer) error { return err } - // t.Blob (blob.Blob) (struct) + // t.Blob (blob.BlobSpec) (struct) if len("blob") > 8192 { return xerrors.Errorf("Value in field \"blob\" was too long") } @@ -124,7 +124,7 @@ func (t *AllocateArguments) UnmarshalCBOR(r io.Reader) (err error) { } switch string(nameBuf[:nameLen]) { - // t.Blob (blob.Blob) (struct) + // t.Blob (blob.BlobSpec) (struct) case "blob": { @@ -312,6 +312,214 @@ func (t *Blob) UnmarshalCBOR(r io.Reader) (err error) { return nil } +func (t *BlobSpecModel) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + fieldCount := 3 + + if t.Digest == nil { + fieldCount-- + } + + if t.DigestCode == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { + return err + } + + // t.Size (uint64) (uint64) + if len("size") > 8192 { + return xerrors.Errorf("Value in field \"size\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("size"))); err != nil { + return err + } + if _, err := cw.WriteString(string("size")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Size)); err != nil { + return err + } + + // t.Digest (multihash.Multihash) (slice) + if t.Digest != nil { + + if len("digest") > 8192 { + return xerrors.Errorf("Value in field \"digest\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("digest"))); err != nil { + return err + } + if _, err := cw.WriteString(string("digest")); err != nil { + return err + } + + if len(t.Digest) > 2097152 { + return xerrors.Errorf("Byte array in field t.Digest was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Digest))); err != nil { + return err + } + + if _, err := cw.Write(t.Digest); err != nil { + return err + } + + } + + // t.DigestCode (uint64) (uint64) + if t.DigestCode != nil { + + if len("digestCode") > 8192 { + return xerrors.Errorf("Value in field \"digestCode\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("digestCode"))); err != nil { + return err + } + if _, err := cw.WriteString(string("digestCode")); err != nil { + return err + } + + if t.DigestCode == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(*t.DigestCode)); err != nil { + return err + } + } + + } + return nil +} + +func (t *BlobSpecModel) UnmarshalCBOR(r io.Reader) (err error) { + *t = BlobSpecModel{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("BlobSpecModel: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 10) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 8192) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Size (uint64) (uint64) + case "size": + + { + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Size = uint64(extra) + + } + // t.Digest (multihash.Multihash) (slice) + case "digest": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 2097152 { + return fmt.Errorf("t.Digest: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.Digest = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.Digest); err != nil { + return err + } + + // t.DigestCode (uint64) (uint64) + case "digestCode": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + typed := uint64(extra) + t.DigestCode = &typed + } + + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} func (t *AllocateOK) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) @@ -718,7 +926,7 @@ func (t *AcceptArguments) MarshalCBOR(w io.Writer) error { return err } - // t.Blob (blob.Blob) (struct) + // t.Blob (blob.BlobSpec) (struct) if len("blob") > 8192 { return xerrors.Errorf("Value in field \"blob\" was too long") } @@ -803,7 +1011,7 @@ func (t *AcceptArguments) UnmarshalCBOR(r io.Reader) (err error) { } } - // t.Blob (blob.Blob) (struct) + // t.Blob (blob.BlobSpec) (struct) case "blob": { @@ -968,7 +1176,7 @@ func (t *AddArguments) MarshalCBOR(w io.Writer) error { return err } - // t.Blob (blob.Blob) (struct) + // t.Blob (blob.BlobSpec) (struct) if len("blob") > 8192 { return xerrors.Errorf("Value in field \"blob\" was too long") } @@ -1027,7 +1235,7 @@ func (t *AddArguments) UnmarshalCBOR(r io.Reader) (err error) { } switch string(nameBuf[:nameLen]) { - // t.Blob (blob.Blob) (struct) + // t.Blob (blob.BlobSpec) (struct) case "blob": { @@ -1429,48 +1637,24 @@ func (t *AbortArguments) MarshalCBOR(w io.Writer) error { cw := cbg.NewCborWriter(w) - if _, err := cw.Write([]byte{162}); err != nil { - return err - } - - // t.Cause (cid.Cid) (struct) - if len("cause") > 8192 { - return xerrors.Errorf("Value in field \"cause\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("cause"))); err != nil { - return err - } - if _, err := cw.WriteString(string("cause")); err != nil { + if _, err := cw.Write([]byte{161}); err != nil { return err } - if err := cbg.WriteCid(cw, t.Cause); err != nil { - return xerrors.Errorf("failed to write cid field t.Cause: %w", err) + // t.Add (cid.Cid) (struct) + if len("add") > 8192 { + return xerrors.Errorf("Value in field \"add\" was too long") } - // t.Digest (multihash.Multihash) (slice) - if len("digest") > 8192 { - return xerrors.Errorf("Value in field \"digest\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("digest"))); err != nil { - return err - } - if _, err := cw.WriteString(string("digest")); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("add"))); err != nil { return err } - - if len(t.Digest) > 2097152 { - return xerrors.Errorf("Byte array in field t.Digest was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Digest))); err != nil { + if _, err := cw.WriteString(string("add")); err != nil { return err } - if _, err := cw.Write(t.Digest); err != nil { - return err + if err := cbg.WriteCid(cw, t.Add); err != nil { + return xerrors.Errorf("failed to write cid field t.Add: %w", err) } return nil @@ -1501,7 +1685,7 @@ func (t *AbortArguments) UnmarshalCBOR(r io.Reader) (err error) { n := extra - nameBuf := make([]byte, 6) + nameBuf := make([]byte, 3) for i := uint64(0); i < n; i++ { nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 8192) if err != nil { @@ -1517,40 +1701,18 @@ func (t *AbortArguments) UnmarshalCBOR(r io.Reader) (err error) { } switch string(nameBuf[:nameLen]) { - // t.Cause (cid.Cid) (struct) - case "cause": + // t.Add (cid.Cid) (struct) + case "add": { c, err := cbg.ReadCid(cr) if err != nil { - return xerrors.Errorf("failed to read cid field t.Cause: %w", err) + return xerrors.Errorf("failed to read cid field t.Add: %w", err) } - t.Cause = c - - } - // t.Digest (multihash.Multihash) (slice) - case "digest": - - maj, extra, err = cr.ReadHeader() - if err != nil { - return err - } - - if extra > 2097152 { - return fmt.Errorf("t.Digest: byte array too large (%d)", extra) - } - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - - if extra > 0 { - t.Digest = make([]uint8, extra) - } + t.Add = c - if _, err := io.ReadFull(cr, t.Digest); err != nil { - return err } default: @@ -1571,48 +1733,24 @@ func (t *RejectArguments) MarshalCBOR(w io.Writer) error { cw := cbg.NewCborWriter(w) - if _, err := cw.Write([]byte{162}); err != nil { + if _, err := cw.Write([]byte{161}); err != nil { return err } - // t.Space (did.DID) (struct) - if len("space") > 8192 { - return xerrors.Errorf("Value in field \"space\" was too long") + // t.Allocation (cid.Cid) (struct) + if len("allocation") > 8192 { + return xerrors.Errorf("Value in field \"allocation\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("space"))); err != nil { - return err - } - if _, err := cw.WriteString(string("space")); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("allocation"))); err != nil { return err } - - if err := t.Space.MarshalCBOR(cw); err != nil { + if _, err := cw.WriteString(string("allocation")); err != nil { return err } - // t.Digest (multihash.Multihash) (slice) - if len("digest") > 8192 { - return xerrors.Errorf("Value in field \"digest\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("digest"))); err != nil { - return err - } - if _, err := cw.WriteString(string("digest")); err != nil { - return err - } - - if len(t.Digest) > 2097152 { - return xerrors.Errorf("Byte array in field t.Digest was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Digest))); err != nil { - return err - } - - if _, err := cw.Write(t.Digest); err != nil { - return err + if err := cbg.WriteCid(cw, t.Allocation); err != nil { + return xerrors.Errorf("failed to write cid field t.Allocation: %w", err) } return nil @@ -1643,7 +1781,7 @@ func (t *RejectArguments) UnmarshalCBOR(r io.Reader) (err error) { n := extra - nameBuf := make([]byte, 6) + nameBuf := make([]byte, 10) for i := uint64(0); i < n; i++ { nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 8192) if err != nil { @@ -1659,37 +1797,18 @@ func (t *RejectArguments) UnmarshalCBOR(r io.Reader) (err error) { } switch string(nameBuf[:nameLen]) { - // t.Space (did.DID) (struct) - case "space": + // t.Allocation (cid.Cid) (struct) + case "allocation": { - if err := t.Space.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.Space: %w", err) + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Allocation: %w", err) } - } - // t.Digest (multihash.Multihash) (slice) - case "digest": - - maj, extra, err = cr.ReadHeader() - if err != nil { - return err - } + t.Allocation = c - if extra > 2097152 { - return fmt.Errorf("t.Digest: byte array too large (%d)", extra) - } - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - - if extra > 0 { - t.Digest = make([]uint8, extra) - } - - if _, err := io.ReadFull(cr, t.Digest); err != nil { - return err } default: diff --git a/commands/blob/codec.go b/commands/blob/codec.go new file mode 100644 index 0000000..febaebc --- /dev/null +++ b/commands/blob/codec.go @@ -0,0 +1,122 @@ +//go:build !codegen + +package blob + +import ( + "fmt" + "io" + + "github.com/fil-forge/ucantone/errors" + "github.com/multiformats/go-multihash" +) + +// UnsupportedDigestCodeErrorName is the stable receipt-failure name when a +// `/blob/add` or `/blob/allocate` names a digest code the executor does not +// support, or the executor does not support adding a blob by digest code at +// all. A client that receives it can fall back to computing the digest before +// adding the blob. +const UnsupportedDigestCodeErrorName = "UnsupportedDigestCode" + +// ErrUnsupportedDigestCode is the failure for an unsupported digest code. +var ErrUnsupportedDigestCode = errors.New(UnsupportedDigestCodeErrorName, "adding a blob by digest code is supported only for sha2-256") + +var ( + errNoVariant = errors.New("InvalidUnion", "union holds no variant") + errBothVariants = errors.New("InvalidUnion", "union holds both variants") +) + +// SpecFromBlob returns the spec of a blob whose digest is known. Its digest +// code is the digest's own; a digest that is not a multihash has code 0. +func SpecFromBlob(b Blob) BlobSpec { + s := BlobSpec{digest: b.Digest, size: b.Size, valid: true} + if d, err := multihash.Decode(b.Digest); err == nil { + s.code = d.Code + } + return s +} + +// SpecFromDigestCode returns the spec of a blob of size bytes whose digest is +// to be computed with the multihash function code. +func SpecFromDigestCode(code, size uint64) BlobSpec { + return BlobSpec{code: code, size: size, valid: true} +} + +// Digest returns the blob's digest, when the spec names one. +func (s BlobSpec) Digest() (multihash.Multihash, bool) { + return s.digest, len(s.digest) > 0 +} + +// DigestCode returns the code of the multihash function the blob's digest is +// computed with: the one the spec names, or its digest's own. +func (s BlobSpec) DigestCode() uint64 { + return s.code +} + +// Size returns the size of the blob. +func (s BlobSpec) Size() uint64 { + return s.size +} + +// model returns the spec's encoded form, which names the digest or the +// digest code but never both. +func (s BlobSpec) model() (BlobSpecModel, error) { + if !s.valid { + return BlobSpecModel{}, errNoVariant + } + if len(s.digest) > 0 { + return BlobSpecModel{Digest: s.digest, Size: s.size}, nil + } + code := s.code + return BlobSpecModel{DigestCode: &code, Size: s.size}, nil +} + +func (s BlobSpec) MarshalCBOR(w io.Writer) error { + m, err := s.model() + if err != nil { + return err + } + return m.MarshalCBOR(w) +} + +func (s *BlobSpec) UnmarshalCBOR(r io.Reader) error { + var m BlobSpecModel + if err := m.UnmarshalCBOR(r); err != nil { + return err + } + return s.fromModel(m) +} + +func (s BlobSpec) MarshalDagJSON(w io.Writer) error { + m, err := s.model() + if err != nil { + return err + } + return m.MarshalDagJSON(w) +} + +func (s *BlobSpec) UnmarshalDagJSON(r io.Reader) error { + var m BlobSpecModel + if err := m.UnmarshalDagJSON(r); err != nil { + return err + } + return s.fromModel(m) +} + +func (s *BlobSpec) fromModel(m BlobSpecModel) error { + *s = BlobSpec{} + switch hasDigest := len(m.Digest) > 0; { + case hasDigest && m.DigestCode == nil: + d, err := multihash.Decode(m.Digest) + if err != nil { + return fmt.Errorf("decoding blob digest: %w", err) + } + *s = BlobSpec{digest: m.Digest, code: d.Code, size: m.Size, valid: true} + case !hasDigest && m.DigestCode != nil: + *s = SpecFromDigestCode(*m.DigestCode, m.Size) + case hasDigest: + return errBothVariants + default: + return errNoVariant + } + return nil +} diff --git a/commands/blob/gen/main.go b/commands/blob/gen/main.go index 76b50a3..d8a22d1 100644 --- a/commands/blob/gen/main.go +++ b/commands/blob/gen/main.go @@ -26,6 +26,7 @@ func main() { models := []any{ blob.AllocateArguments{}, blob.Blob{}, + blob.BlobSpecModel{}, blob.AllocateOK{}, blob.BlobAddress{}, blob.AcceptArguments{}, diff --git a/commands/blob/json_gen.go b/commands/blob/json_gen.go index 11d0010..dc8690e 100644 --- a/commands/blob/json_gen.go +++ b/commands/blob/json_gen.go @@ -32,7 +32,7 @@ func (t *AllocateArguments) MarshalDagJSON(w io.Writer) error { } written := false - // t.Blob (blob.Blob) (struct) + // t.Blob (blob.BlobSpec) (struct) if len("blob") > 8192 { return fmt.Errorf("string in field \"blob\" was too long") } @@ -126,7 +126,7 @@ func (t *AllocateArguments) UnmarshalDagJSON(r io.Reader) (err error) { } switch name { - // t.Blob (blob.Blob) (struct) + // t.Blob (blob.BlobSpec) (struct) case "blob": if err := t.Blob.UnmarshalDagJSON(jr); err != nil { @@ -313,6 +313,194 @@ func (t *Blob) UnmarshalDagJSON(r io.Reader) (err error) { return nil } +func (t *BlobSpecModel) MarshalDagJSON(w io.Writer) error { + jw := jsg.NewDagJsonWriter(w) + if t == nil { + err := jw.WriteNull() + return err + } + if err := jw.WriteObjectOpen(); err != nil { + return err + } + written := false + + // t.Digest (multihash.Multihash) (slice) + if t.Digest != nil { + if len("digest") > 8192 { + return fmt.Errorf("string in field \"digest\" was too long") + } + if err := jw.WriteString(string("digest")); err != nil { + return fmt.Errorf("writing string for field \"digest\": %w", err) + } + if err := jw.WriteObjectColon(); err != nil { + return err + } + if len(t.Digest) > 2097152 { + return fmt.Errorf("byte array in field t.Digest was too long") + } + + if err := jw.WriteBytes(t.Digest); err != nil { + return fmt.Errorf("writing bytes for field t.Digest: %w", err) + } + + written = true + } + if t.DigestCode != nil { + if written { + if err := jw.WriteComma(); err != nil { + return err + } + } + } + + // t.DigestCode (uint64) (uint64) + if t.DigestCode != nil { + if len("digestCode") > 8192 { + return fmt.Errorf("string in field \"digestCode\" was too long") + } + if err := jw.WriteString(string("digestCode")); err != nil { + return fmt.Errorf("writing string for field \"digestCode\": %w", err) + } + if err := jw.WriteObjectColon(); err != nil { + return err + } + + if t.DigestCode == nil { + if err := jw.WriteNull(); err != nil { + return fmt.Errorf("writing null for field t.DigestCode: %w", err) + } + } else { + if err := jw.WriteUint64(uint64(*t.DigestCode)); err != nil { + return fmt.Errorf("writing uint64 for field t.DigestCode: %w", err) + } + } + + written = true + } + if written { + if err := jw.WriteComma(); err != nil { + return err + } + } + + // t.Size (uint64) (uint64) + if len("size") > 8192 { + return fmt.Errorf("string in field \"size\" was too long") + } + if err := jw.WriteString(string("size")); err != nil { + return fmt.Errorf("writing string for field \"size\": %w", err) + } + if err := jw.WriteObjectColon(); err != nil { + return err + } + + if err := jw.WriteUint64(uint64(t.Size)); err != nil { + return fmt.Errorf("writing uint64 for field t.Size: %w", err) + } + + if err := jw.WriteObjectClose(); err != nil { + return err + } + return nil +} +func (t *BlobSpecModel) UnmarshalDagJSON(r io.Reader) (err error) { + *t = BlobSpecModel{} + + jr := jsg.NewDagJsonReader(r) + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if err := jr.ReadObjectOpen(); err != nil { + return fmt.Errorf("reading object open for BlobSpecModel: %w", err) + } + close, err := jr.PeekObjectClose() + if err != nil { + return fmt.Errorf("peeking object close for BlobSpecModel: %w", err) + } + if close { + if err := jr.ReadObjectClose(); err != nil { + return fmt.Errorf("reading object close for BlobSpecModel: %w", err) + } + } else { + for i := uint64(0); i < 8192; i++ { + name, err := jr.ReadString(8192) + if err != nil { + if errors.Is(err, jsg.ErrLimitExceeded) { + return fmt.Errorf("reading string for field BlobSpecModel: string too large") + } + return fmt.Errorf("reading string for field BlobSpecModel: %w", err) + } + if err := jr.ReadObjectColon(); err != nil { + return fmt.Errorf("reading object colon for field BlobSpecModel: %w", err) + } + switch name { + + // t.Digest (multihash.Multihash) (slice) + case "digest": + + { + bval, err := jr.ReadBytes(2097152) + if err != nil { + if errors.Is(err, jsg.ErrLimitExceeded) { + return fmt.Errorf("reading bytes for field t.Digest: byte array too large") + } + return fmt.Errorf("reading bytes for field t.Digest: %w", err) + } + if len(bval) > 0 { + t.Digest = []uint8(bval) + } + } + + // t.DigestCode (uint64) (uint64) + case "digestCode": + { + + nval, err := jr.ReadNumberAsUint64OrNull() + if err != nil { + return fmt.Errorf("reading uint64 or null for field t.DigestCode: %w", err) + } + if nval != nil { + typed := uint64(*nval) + t.DigestCode = &typed + } + + } + + // t.Size (uint64) (uint64) + case "size": + { + + nval, err := jr.ReadNumberAsUint64() + if err != nil { + return fmt.Errorf("reading uint64 for field t.Size: %w", err) + } + t.Size = uint64(nval) + + } + default: + // Field doesn't exist on this type, so ignore it + if err := jr.DiscardType(); err != nil { + return fmt.Errorf("ignoring field %s for BlobSpecModel: %w", name, err) + } + } + + close, err := jr.ReadObjectCloseOrComma() + if err != nil { + return fmt.Errorf("reading object close or comma for field BlobSpecModel: %w", err) + } + if close { + break + } + if i == 8192-1 { + return fmt.Errorf("map too large for BlobSpecModel") + } + } + } + + return nil +} func (t *AllocateOK) MarshalDagJSON(w io.Writer) error { jw := jsg.NewDagJsonWriter(w) if t == nil { @@ -724,7 +912,7 @@ func (t *AcceptArguments) MarshalDagJSON(w io.Writer) error { } } - // t.Blob (blob.Blob) (struct) + // t.Blob (blob.BlobSpec) (struct) if len("blob") > 8192 { return fmt.Errorf("string in field \"blob\" was too long") } @@ -803,7 +991,7 @@ func (t *AcceptArguments) UnmarshalDagJSON(r io.Reader) (err error) { return fmt.Errorf("unmarshaling t.Put: %w", err) } - // t.Blob (blob.Blob) (struct) + // t.Blob (blob.BlobSpec) (struct) case "blob": if err := t.Blob.UnmarshalDagJSON(jr); err != nil { @@ -974,7 +1162,7 @@ func (t *AddArguments) MarshalDagJSON(w io.Writer) error { return err } - // t.Blob (blob.Blob) (struct) + // t.Blob (blob.BlobSpec) (struct) if len("blob") > 8192 { return fmt.Errorf("string in field \"blob\" was too long") } @@ -1026,7 +1214,7 @@ func (t *AddArguments) UnmarshalDagJSON(r io.Reader) (err error) { } switch name { - // t.Blob (blob.Blob) (struct) + // t.Blob (blob.BlobSpec) (struct) case "blob": if err := t.Blob.UnmarshalDagJSON(jr); err != nil { @@ -1427,46 +1615,20 @@ func (t *AbortArguments) MarshalDagJSON(w io.Writer) error { if err := jw.WriteObjectOpen(); err != nil { return err } - written := false - - // t.Cause (cid.Cid) (struct) - if len("cause") > 8192 { - return fmt.Errorf("string in field \"cause\" was too long") - } - if err := jw.WriteString(string("cause")); err != nil { - return fmt.Errorf("writing string for field \"cause\": %w", err) - } - if err := jw.WriteObjectColon(); err != nil { - return err - } - if err := jw.WriteCid(t.Cause); err != nil { - return fmt.Errorf("writing CID for field t.Cause: %w", err) + // t.Add (cid.Cid) (struct) + if len("add") > 8192 { + return fmt.Errorf("string in field \"add\" was too long") } - - written = true - if written { - if err := jw.WriteComma(); err != nil { - return err - } - } - - // t.Digest (multihash.Multihash) (slice) - if len("digest") > 8192 { - return fmt.Errorf("string in field \"digest\" was too long") - } - if err := jw.WriteString(string("digest")); err != nil { - return fmt.Errorf("writing string for field \"digest\": %w", err) + if err := jw.WriteString(string("add")); err != nil { + return fmt.Errorf("writing string for field \"add\": %w", err) } if err := jw.WriteObjectColon(); err != nil { return err } - if len(t.Digest) > 2097152 { - return fmt.Errorf("byte array in field t.Digest was too long") - } - if err := jw.WriteBytes(t.Digest); err != nil { - return fmt.Errorf("writing bytes for field t.Digest: %w", err) + if err := jw.WriteCid(t.Add); err != nil { + return fmt.Errorf("writing CID for field t.Add: %w", err) } if err := jw.WriteObjectClose(); err != nil { @@ -1508,34 +1670,17 @@ func (t *AbortArguments) UnmarshalDagJSON(r io.Reader) (err error) { } switch name { - // t.Cause (cid.Cid) (struct) - case "cause": + // t.Add (cid.Cid) (struct) + case "add": { c, err := jr.ReadCid() if err != nil { - return fmt.Errorf("reading CID for field t.Cause: %w", err) + return fmt.Errorf("reading CID for field t.Add: %w", err) } - t.Cause = c + t.Add = c } - - // t.Digest (multihash.Multihash) (slice) - case "digest": - - { - bval, err := jr.ReadBytes(2097152) - if err != nil { - if errors.Is(err, jsg.ErrLimitExceeded) { - return fmt.Errorf("reading bytes for field t.Digest: byte array too large") - } - return fmt.Errorf("reading bytes for field t.Digest: %w", err) - } - if len(bval) > 0 { - t.Digest = []uint8(bval) - } - } - default: // Field doesn't exist on this type, so ignore it if err := jr.DiscardType(); err != nil { @@ -1567,46 +1712,22 @@ func (t *RejectArguments) MarshalDagJSON(w io.Writer) error { if err := jw.WriteObjectOpen(); err != nil { return err } - written := false - // t.Digest (multihash.Multihash) (slice) - if len("digest") > 8192 { - return fmt.Errorf("string in field \"digest\" was too long") + // t.Allocation (cid.Cid) (struct) + if len("allocation") > 8192 { + return fmt.Errorf("string in field \"allocation\" was too long") } - if err := jw.WriteString(string("digest")); err != nil { - return fmt.Errorf("writing string for field \"digest\": %w", err) + if err := jw.WriteString(string("allocation")); err != nil { + return fmt.Errorf("writing string for field \"allocation\": %w", err) } if err := jw.WriteObjectColon(); err != nil { return err } - if len(t.Digest) > 2097152 { - return fmt.Errorf("byte array in field t.Digest was too long") - } - - if err := jw.WriteBytes(t.Digest); err != nil { - return fmt.Errorf("writing bytes for field t.Digest: %w", err) - } - written = true - if written { - if err := jw.WriteComma(); err != nil { - return err - } + if err := jw.WriteCid(t.Allocation); err != nil { + return fmt.Errorf("writing CID for field t.Allocation: %w", err) } - // t.Space (did.DID) (struct) - if len("space") > 8192 { - return fmt.Errorf("string in field \"space\" was too long") - } - if err := jw.WriteString(string("space")); err != nil { - return fmt.Errorf("writing string for field \"space\": %w", err) - } - if err := jw.WriteObjectColon(); err != nil { - return err - } - if err := t.Space.MarshalDagJSON(jw); err != nil { - return fmt.Errorf("marshaling field t.Space: %w", err) - } if err := jw.WriteObjectClose(); err != nil { return err } @@ -1646,29 +1767,17 @@ func (t *RejectArguments) UnmarshalDagJSON(r io.Reader) (err error) { } switch name { - // t.Digest (multihash.Multihash) (slice) - case "digest": - + // t.Allocation (cid.Cid) (struct) + case "allocation": { - bval, err := jr.ReadBytes(2097152) + + c, err := jr.ReadCid() if err != nil { - if errors.Is(err, jsg.ErrLimitExceeded) { - return fmt.Errorf("reading bytes for field t.Digest: byte array too large") - } - return fmt.Errorf("reading bytes for field t.Digest: %w", err) - } - if len(bval) > 0 { - t.Digest = []uint8(bval) + return fmt.Errorf("reading CID for field t.Allocation: %w", err) } - } - - // t.Space (did.DID) (struct) - case "space": + t.Allocation = c - if err := t.Space.UnmarshalDagJSON(jr); err != nil { - return fmt.Errorf("unmarshaling t.Space: %w", err) } - default: // Field doesn't exist on this type, so ignore it if err := jr.DiscardType(); err != nil { diff --git a/commands/blob/reject.go b/commands/blob/reject.go index 9abf29e..b2ac517 100644 --- a/commands/blob/reject.go +++ b/commands/blob/reject.go @@ -19,15 +19,16 @@ type RejectOK = commands.Unit // // Served by storage nodes (subject = the provider DID, invoked by the // upload service under its registration delegation, typically translating a -// client `/blob/abort`). The node drops the space's allocation and deletes -// any received bytes once no space holds an allocation or acceptance for -// the digest. +// client `/blob/abort`). The allocation is named by the `/blob/allocate` task +// that made it, and the node knows which space and blob it is for. The node +// drops the allocation and deletes any received bytes once no space holds an +// allocation or acceptance for them. // -// A blob that THE INVOKING SPACE has accepted is refused with BlobAccepted — -// a space's accepted blobs are released via `/blob/remove`, never rejected. -// The guard is scoped to the invoking space, not the digest: another space's -// acceptance of the same bytes must not block the reject — the node simply -// drops this space's allocation and retains the bytes for the space that +// A blob that the allocation's space has accepted is refused with +// BlobAccepted — a space's accepted blobs are released via `/blob/remove`, +// never rejected. The guard is scoped to that space, not the digest: another +// space's acceptance of the same bytes must not block the reject — the node +// simply drops this allocation and retains the bytes for the space that // still claims them. // // Idempotent: rejecting an unknown or already-rejected blob succeeds. diff --git a/commands/blob/spec_test.go b/commands/blob/spec_test.go new file mode 100644 index 0000000..629aded --- /dev/null +++ b/commands/blob/spec_test.go @@ -0,0 +1,152 @@ +//go:build !codegen + +package blob_test + +import ( + "bytes" + "strings" + "testing" + + "github.com/fil-forge/libforge/commands/blob" + "github.com/fil-forge/libforge/testutil" + "github.com/multiformats/go-multicodec" + "github.com/stretchr/testify/require" +) + +// A spec with a digest encodes exactly as a Blob does, so invocations that +// name a digest keep their task links. +func TestBlobSpecWithDigestEncodesAsBlob(t *testing.T) { + b := blob.Blob{Digest: testutil.RandomMultihash(t), Size: 1024} + spec := blob.SpecFromBlob(b) + + var wantCBOR, gotCBOR bytes.Buffer + require.NoError(t, b.MarshalCBOR(&wantCBOR)) + require.NoError(t, spec.MarshalCBOR(&gotCBOR)) + require.Equal(t, wantCBOR.Bytes(), gotCBOR.Bytes()) + + var wantJSON, gotJSON bytes.Buffer + require.NoError(t, b.MarshalDagJSON(&wantJSON)) + require.NoError(t, spec.MarshalDagJSON(&gotJSON)) + require.Equal(t, wantJSON.String(), gotJSON.String()) +} + +func TestBlobSpecRoundTrip(t *testing.T) { + b := blob.Blob{Digest: testutil.RandomMultihash(t), Size: 7} + code := uint64(multicodec.Sha2_256) + for name, in := range map[string]blob.BlobSpec{ + "digest": blob.SpecFromBlob(b), + "digest code": blob.SpecFromDigestCode(code, 2097152), + } { + t.Run(name, func(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, in.MarshalCBOR(&buf)) + var out blob.BlobSpec + require.NoError(t, out.UnmarshalCBOR(&buf)) + require.Equal(t, in, out) + + var js bytes.Buffer + require.NoError(t, in.MarshalDagJSON(&js)) + var outJS blob.BlobSpec + require.NoError(t, outJS.UnmarshalDagJSON(&js)) + require.Equal(t, in, outJS) + }) + } + +} + +func TestBlobSpecAccessors(t *testing.T) { + digest := testutil.RandomMultihash(t) + byDigest := blob.SpecFromBlob(blob.Blob{Digest: digest, Size: 7}) + got, ok := byDigest.Digest() + require.True(t, ok) + require.Equal(t, digest, got) + require.Equal(t, uint64(multicodec.Sha2_256), byDigest.DigestCode(), "a digest has its own code") + require.EqualValues(t, 7, byDigest.Size()) + + byCode := blob.SpecFromDigestCode(uint64(multicodec.Sha2_256), 9) + _, ok = byCode.Digest() + require.False(t, ok) + require.Equal(t, uint64(multicodec.Sha2_256), byCode.DigestCode()) + require.EqualValues(t, 9, byCode.Size()) +} + +// A spec naming a digest decodes only when the digest is a multihash, since +// its digest code comes from it. +func TestBlobSpecRefusesMalformedDigest(t *testing.T) { + m := blob.BlobSpecModel{Digest: []byte{0xff}, Size: 1} + var buf bytes.Buffer + require.NoError(t, m.MarshalCBOR(&buf)) + var out blob.BlobSpec + require.Error(t, out.UnmarshalCBOR(&buf)) +} + +// A spec travels inside the arguments' generated codecs: the nested union +// round-trips in both encodings. +func TestAddArgumentsWithDigestCodeRoundTrip(t *testing.T) { + in := blob.AddArguments{Blob: blob.SpecFromDigestCode(uint64(multicodec.Sha2_256), 42)} + var buf bytes.Buffer + require.NoError(t, in.MarshalCBOR(&buf)) + var out blob.AddArguments + require.NoError(t, out.UnmarshalCBOR(&buf)) + require.Equal(t, in, out) + + var js bytes.Buffer + require.NoError(t, in.MarshalDagJSON(&js)) + require.Equal(t, `{"blob":{"digestCode":18,"size":42}}`, js.String()) + var outJS blob.AddArguments + require.NoError(t, outJS.UnmarshalDagJSON(&js)) + require.Equal(t, in, outJS) +} + +// A spec holding neither variant cannot be encoded, and an encoding holding +// neither or both cannot be decoded. +func TestBlobSpecRejectsInvalidUnion(t *testing.T) { + var buf bytes.Buffer + require.Error(t, blob.BlobSpec{}.MarshalCBOR(&buf)) + require.Error(t, blob.BlobSpec{}.MarshalDagJSON(&buf)) + + for name, js := range map[string]string{ + "neither": `{"size":1}`, + "both": `{"digest":{"/":{"bytes":"EiA"}},"digestCode":18,"size":1}`, + } { + t.Run(name, func(t *testing.T) { + var out blob.BlobSpec + require.Error(t, out.UnmarshalDagJSON(strings.NewReader(js))) + }) + } +} + +// A spec decodes through its model, which holds the fields of both variants: +// exactly one variant's field must be set. +func TestBlobSpecModelRejectsInvalidCBOR(t *testing.T) { + digest := testutil.RandomMultihash(t) + code := uint64(multicodec.Sha2_256) + + for name, m := range map[string]blob.BlobSpecModel{ + "neither": {Size: 1}, + "both": {Digest: digest, DigestCode: &code, Size: 1}, + } { + t.Run("spec "+name, func(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, m.MarshalCBOR(&buf)) + var out blob.BlobSpec + require.Error(t, out.UnmarshalCBOR(&buf)) + }) + } +} + +// The model's generated decoder bounds each field as it reads it, so a spec +// decodes in one pass with no intermediate copy of its fields. +func TestBlobSpecRefusesOversizedDigest(t *testing.T) { + // {"digest": <3 MiB of bytes>, "size": 1}, written by hand: the generated + // encoder refuses a digest this large too. + n := 3 << 20 + var buf bytes.Buffer + buf.WriteByte(0xa2) + buf.WriteString("\x66digest") + buf.Write([]byte{0x5a, byte(n >> 24), byte(n >> 16), byte(n >> 8), byte(n)}) + buf.Write(bytes.Repeat([]byte{1}, n)) + buf.WriteString("\x64size\x01") + var out blob.BlobSpec + require.ErrorContains(t, out.UnmarshalCBOR(&buf), "t.Digest: byte array too large") +} diff --git a/commands/blob/types.go b/commands/blob/types.go index 27cb5ca..5a439b9 100644 --- a/commands/blob/types.go +++ b/commands/blob/types.go @@ -13,8 +13,36 @@ type Blob struct { Size uint64 `cborgen:"size" dagjsongen:"size"` } +// BlobSpec describes a blob in the arguments of the capabilities that can +// add a blob before its digest is known: `/blob/add`, `/blob/allocate`, +// `/blob/accept` and the body of `/http/put`. It names the blob's size and +// either its digest, when the digest is known in advance, or only the code of +// the multihash function its digest is to be computed with, when both parties +// compute it as the data streams. A spec naming a digest has the digest's +// code too. +// +// Construct one with [SpecFromBlob] or [SpecFromDigestCode]. Encoding the zero +// value fails, as does decoding a spec that names both a digest and a digest +// code, or neither. The codec is hand-written (codec.go), over the generated +// codec of [BlobSpecModel]. +type BlobSpec struct { + digest multihash.Multihash + code uint64 + size uint64 + valid bool +} + +// BlobSpecModel is the encoded form of a [BlobSpec]: the fields of both of its +// variants, so a spec decodes in one bounded pass. Exactly one of Digest and +// DigestCode is set in a valid spec. +type BlobSpecModel struct { + Digest multihash.Multihash `cborgen:"digest,omitempty" dagjsongen:"digest,omitempty"` + DigestCode *uint64 `cborgen:"digestCode,omitempty" dagjsongen:"digestCode,omitempty"` + Size uint64 `cborgen:"size" dagjsongen:"size"` +} + type AddArguments struct { - Blob Blob `cborgen:"blob" dagjsongen:"blob"` + Blob BlobSpec `cborgen:"blob" dagjsongen:"blob"` } type AddOK struct { @@ -25,7 +53,7 @@ type AddOK struct { type AcceptArguments struct { Space did.DID `cborgen:"space" dagjsongen:"space"` - Blob Blob `cborgen:"blob" dagjsongen:"blob"` + Blob BlobSpec `cborgen:"blob" dagjsongen:"blob"` Put promise.AwaitOK `cborgen:"_put" dagjsongen:"_put"` } @@ -37,9 +65,9 @@ type AcceptOK struct { } type AllocateArguments struct { - Space did.DID `cborgen:"space" dagjsongen:"space"` - Blob Blob `cborgen:"blob" dagjsongen:"blob"` - Cause cid.Cid `cborgen:"cause" dagjsongen:"cause"` + Space did.DID `cborgen:"space" dagjsongen:"space"` + Blob BlobSpec `cborgen:"blob" dagjsongen:"blob"` + Cause cid.Cid `cborgen:"cause" dagjsongen:"cause"` } type AllocateOK struct { @@ -95,23 +123,22 @@ type ReleaseArguments struct { Cause cid.Cid `cborgen:"cause" dagjsongen:"cause"` } -// AbortArguments abandons the invoking space's in-flight upload of the -// parked (never-accepted) blob identified by Digest. The space is the -// invocation subject. Cause is the `/blob/add` task link: the upload -// service uses it to recover which storage node holds the parked blob — a -// parked blob has no registration or acceptance to look the node up by. +// AbortArguments abandons the space's in-flight upload of a parked +// (never-accepted) blob. The space is the invocation subject. Add is the +// `/blob/add` task link: the upload service follows its receipt chain to the +// storage node holding the upload and the allocation it made there. A parked +// blob has no registration or acceptance to look the node up by. type AbortArguments struct { - Digest multihash.Multihash `cborgen:"digest" dagjsongen:"digest"` - Cause cid.Cid `cborgen:"cause" dagjsongen:"cause"` + Add cid.Cid `cborgen:"add" dagjsongen:"add"` } -// RejectArguments drops Space's allocation for the parked (never-accepted) -// blob identified by Digest on the storage node; the node deletes any -// received bytes once no space holds an allocation or acceptance for the -// digest. +// RejectArguments drops the allocation Allocation names, the link to the +// `/blob/allocate` task that made it, for a parked (never-accepted) blob on +// the storage node. The node knows the space and blob the allocation is for, +// and deletes any received bytes once no space holds an allocation or +// acceptance for them. type RejectArguments struct { - Space did.DID `cborgen:"space" dagjsongen:"space"` - Digest multihash.Multihash `cborgen:"digest" dagjsongen:"digest"` + Allocation cid.Cid `cborgen:"allocation" dagjsongen:"allocation"` } type ReplicateArguments struct { diff --git a/commands/http/cbor_gen.go b/commands/http/cbor_gen.go index da88658..0ea25cc 100644 --- a/commands/http/cbor_gen.go +++ b/commands/http/cbor_gen.go @@ -32,7 +32,7 @@ func (t *PutArguments) MarshalCBOR(w io.Writer) error { return err } - // t.Body (blob.Blob) (struct) + // t.Body (blob.BlobSpec) (struct) if len("body") > 8192 { return xerrors.Errorf("Value in field \"body\" was too long") } @@ -107,7 +107,7 @@ func (t *PutArguments) UnmarshalCBOR(r io.Reader) (err error) { } switch string(nameBuf[:nameLen]) { - // t.Body (blob.Blob) (struct) + // t.Body (blob.BlobSpec) (struct) case "body": { @@ -138,3 +138,226 @@ func (t *PutArguments) UnmarshalCBOR(r io.Reader) (err error) { return nil } +func (t *PutOK) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + fieldCount := 1 + + if t.Blob == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { + return err + } + + // t.Blob (http.PutBlob) (struct) + if t.Blob != nil { + + if len("blob") > 8192 { + return xerrors.Errorf("Value in field \"blob\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("blob"))); err != nil { + return err + } + if _, err := cw.WriteString(string("blob")); err != nil { + return err + } + + if err := t.Blob.MarshalCBOR(cw); err != nil { + return err + } + } + return nil +} + +func (t *PutOK) UnmarshalCBOR(r io.Reader) (err error) { + *t = PutOK{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("PutOK: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 4) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 8192) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Blob (http.PutBlob) (struct) + case "blob": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Blob = new(PutBlob) + if err := t.Blob.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Blob pointer: %w", err) + } + } + + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *PutBlob) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{161}); err != nil { + return err + } + + // t.Digest (multihash.Multihash) (slice) + if len("digest") > 8192 { + return xerrors.Errorf("Value in field \"digest\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("digest"))); err != nil { + return err + } + if _, err := cw.WriteString(string("digest")); err != nil { + return err + } + + if len(t.Digest) > 2097152 { + return xerrors.Errorf("Byte array in field t.Digest was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Digest))); err != nil { + return err + } + + if _, err := cw.Write(t.Digest); err != nil { + return err + } + + return nil +} + +func (t *PutBlob) UnmarshalCBOR(r io.Reader) (err error) { + *t = PutBlob{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("PutBlob: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 6) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 8192) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Digest (multihash.Multihash) (slice) + case "digest": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 2097152 { + return fmt.Errorf("t.Digest: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.Digest = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.Digest); err != nil { + return err + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} diff --git a/commands/http/gen/main.go b/commands/http/gen/main.go index b12a0a4..b009b97 100644 --- a/commands/http/gen/main.go +++ b/commands/http/gen/main.go @@ -25,6 +25,8 @@ func tag(path string) { func main() { models := []any{ http.PutArguments{}, + http.PutOK{}, + http.PutBlob{}, } const ( cborFile = "../cbor_gen.go" diff --git a/commands/http/json_gen.go b/commands/http/json_gen.go index 61d152b..539f562 100644 --- a/commands/http/json_gen.go +++ b/commands/http/json_gen.go @@ -31,7 +31,7 @@ func (t *PutArguments) MarshalDagJSON(w io.Writer) error { } written := false - // t.Body (blob.Blob) (struct) + // t.Body (blob.BlobSpec) (struct) if len("body") > 8192 { return fmt.Errorf("string in field \"body\" was too long") } @@ -103,7 +103,7 @@ func (t *PutArguments) UnmarshalDagJSON(r io.Reader) (err error) { } switch name { - // t.Body (blob.Blob) (struct) + // t.Body (blob.BlobSpec) (struct) case "body": if err := t.Body.UnmarshalDagJSON(jr); err != nil { @@ -139,3 +139,214 @@ func (t *PutArguments) UnmarshalDagJSON(r io.Reader) (err error) { return nil } +func (t *PutOK) MarshalDagJSON(w io.Writer) error { + jw := jsg.NewDagJsonWriter(w) + if t == nil { + err := jw.WriteNull() + return err + } + if err := jw.WriteObjectOpen(); err != nil { + return err + } + + // t.Blob (http.PutBlob) (struct) + if t.Blob != nil { + if len("blob") > 8192 { + return fmt.Errorf("string in field \"blob\" was too long") + } + if err := jw.WriteString(string("blob")); err != nil { + return fmt.Errorf("writing string for field \"blob\": %w", err) + } + if err := jw.WriteObjectColon(); err != nil { + return err + } + if err := t.Blob.MarshalDagJSON(jw); err != nil { + return fmt.Errorf("marshaling field t.Blob: %w", err) + } + } + if err := jw.WriteObjectClose(); err != nil { + return err + } + return nil +} +func (t *PutOK) UnmarshalDagJSON(r io.Reader) (err error) { + *t = PutOK{} + + jr := jsg.NewDagJsonReader(r) + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if err := jr.ReadObjectOpen(); err != nil { + return fmt.Errorf("reading object open for PutOK: %w", err) + } + close, err := jr.PeekObjectClose() + if err != nil { + return fmt.Errorf("peeking object close for PutOK: %w", err) + } + if close { + if err := jr.ReadObjectClose(); err != nil { + return fmt.Errorf("reading object close for PutOK: %w", err) + } + } else { + for i := uint64(0); i < 8192; i++ { + name, err := jr.ReadString(8192) + if err != nil { + if errors.Is(err, jsg.ErrLimitExceeded) { + return fmt.Errorf("reading string for field PutOK: string too large") + } + return fmt.Errorf("reading string for field PutOK: %w", err) + } + if err := jr.ReadObjectColon(); err != nil { + return fmt.Errorf("reading object colon for field PutOK: %w", err) + } + switch name { + + // t.Blob (http.PutBlob) (struct) + case "blob": + + { + null, err := jr.PeekNull() + if err != nil { + return fmt.Errorf("peeking null for field t.Blob: %w", err) + } + if null { + if err := jr.ReadNull(); err != nil { + return fmt.Errorf("reading null for field t.Blob: %w", err) + } + } else { + t.Blob = new(PutBlob) + if err := t.Blob.UnmarshalDagJSON(jr); err != nil { + return fmt.Errorf("unmarshaling t.Blob pointer: %w", err) + } + } + } + + default: + // Field doesn't exist on this type, so ignore it + if err := jr.DiscardType(); err != nil { + return fmt.Errorf("ignoring field %s for PutOK: %w", name, err) + } + } + + close, err := jr.ReadObjectCloseOrComma() + if err != nil { + return fmt.Errorf("reading object close or comma for field PutOK: %w", err) + } + if close { + break + } + if i == 8192-1 { + return fmt.Errorf("map too large for PutOK") + } + } + } + + return nil +} +func (t *PutBlob) MarshalDagJSON(w io.Writer) error { + jw := jsg.NewDagJsonWriter(w) + if t == nil { + err := jw.WriteNull() + return err + } + if err := jw.WriteObjectOpen(); err != nil { + return err + } + + // t.Digest (multihash.Multihash) (slice) + if len("digest") > 8192 { + return fmt.Errorf("string in field \"digest\" was too long") + } + if err := jw.WriteString(string("digest")); err != nil { + return fmt.Errorf("writing string for field \"digest\": %w", err) + } + if err := jw.WriteObjectColon(); err != nil { + return err + } + if len(t.Digest) > 2097152 { + return fmt.Errorf("byte array in field t.Digest was too long") + } + + if err := jw.WriteBytes(t.Digest); err != nil { + return fmt.Errorf("writing bytes for field t.Digest: %w", err) + } + + if err := jw.WriteObjectClose(); err != nil { + return err + } + return nil +} +func (t *PutBlob) UnmarshalDagJSON(r io.Reader) (err error) { + *t = PutBlob{} + + jr := jsg.NewDagJsonReader(r) + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if err := jr.ReadObjectOpen(); err != nil { + return fmt.Errorf("reading object open for PutBlob: %w", err) + } + close, err := jr.PeekObjectClose() + if err != nil { + return fmt.Errorf("peeking object close for PutBlob: %w", err) + } + if close { + if err := jr.ReadObjectClose(); err != nil { + return fmt.Errorf("reading object close for PutBlob: %w", err) + } + } else { + for i := uint64(0); i < 8192; i++ { + name, err := jr.ReadString(8192) + if err != nil { + if errors.Is(err, jsg.ErrLimitExceeded) { + return fmt.Errorf("reading string for field PutBlob: string too large") + } + return fmt.Errorf("reading string for field PutBlob: %w", err) + } + if err := jr.ReadObjectColon(); err != nil { + return fmt.Errorf("reading object colon for field PutBlob: %w", err) + } + switch name { + + // t.Digest (multihash.Multihash) (slice) + case "digest": + + { + bval, err := jr.ReadBytes(2097152) + if err != nil { + if errors.Is(err, jsg.ErrLimitExceeded) { + return fmt.Errorf("reading bytes for field t.Digest: byte array too large") + } + return fmt.Errorf("reading bytes for field t.Digest: %w", err) + } + if len(bval) > 0 { + t.Digest = []uint8(bval) + } + } + + default: + // Field doesn't exist on this type, so ignore it + if err := jr.DiscardType(); err != nil { + return fmt.Errorf("ignoring field %s for PutBlob: %w", name, err) + } + } + + close, err := jr.ReadObjectCloseOrComma() + if err != nil { + return fmt.Errorf("reading object close or comma for field PutBlob: %w", err) + } + if close { + break + } + if i == 8192-1 { + return fmt.Errorf("map too large for PutBlob") + } + } + } + + return nil +} diff --git a/commands/http/put.go b/commands/http/put.go index 91431c5..00f4d61 100644 --- a/commands/http/put.go +++ b/commands/http/put.go @@ -3,11 +3,8 @@ package http import ( - "github.com/fil-forge/libforge/commands" "github.com/fil-forge/ucantone/binding" "github.com/fil-forge/ucantone/ucan/command" ) -type PutOK = commands.Unit - var Put = binding.Bind[*PutArguments, *PutOK](command.MustParse("/http/put")) diff --git a/commands/http/put_test.go b/commands/http/put_test.go new file mode 100644 index 0000000..fe6c38d --- /dev/null +++ b/commands/http/put_test.go @@ -0,0 +1,41 @@ +//go:build !codegen + +package http_test + +import ( + "bytes" + "testing" + + "github.com/fil-forge/libforge/commands" + "github.com/fil-forge/libforge/commands/http" + "github.com/fil-forge/libforge/testutil" + "github.com/stretchr/testify/require" +) + +func TestPutOKRoundTrip(t *testing.T) { + in := http.PutOK{Blob: &http.PutBlob{Digest: testutil.RandomMultihash(t)}} + var buf bytes.Buffer + require.NoError(t, in.MarshalCBOR(&buf)) + var out http.PutOK + require.NoError(t, out.UnmarshalCBOR(&buf)) + require.Equal(t, in, out) + + var js bytes.Buffer + require.NoError(t, in.MarshalDagJSON(&js)) + var outJS http.PutOK + require.NoError(t, outJS.UnmarshalDagJSON(&js)) + require.Equal(t, in, outJS) +} + +// An empty result encodes as the empty map it was before it could carry a +// digest, so existing receipts decode and encode unchanged. +func TestPutOKEmptyEncodesAsUnit(t *testing.T) { + var want, got bytes.Buffer + require.NoError(t, (&commands.Unit{}).MarshalCBOR(&want)) + require.NoError(t, (&http.PutOK{}).MarshalCBOR(&got)) + require.Equal(t, want.Bytes(), got.Bytes()) + + var out http.PutOK + require.NoError(t, out.UnmarshalCBOR(bytes.NewReader(want.Bytes()))) + require.Nil(t, out.Blob) +} diff --git a/commands/http/types.go b/commands/http/types.go index 22e2084..29f680f 100644 --- a/commands/http/types.go +++ b/commands/http/types.go @@ -3,11 +3,25 @@ package http import ( "github.com/fil-forge/libforge/commands/blob" "github.com/fil-forge/ucantone/ucan/promise" + "github.com/multiformats/go-multihash" ) type PutArguments struct { - Body blob.Blob `cborgen:"body" dagjsongen:"body"` + Body blob.BlobSpec `cborgen:"body" dagjsongen:"body"` // Destination is the promise that resolves to the upload destination // where the blob should be PUT to. It is the result of a /blob/allocate task. Destination promise.AwaitOK `cborgen:"destination" dagjsongen:"destination"` } + +// PutOK is the result of a successful `/http/put`. Blob is set when the +// `/blob/allocate` task named only a digest code: it carries the digest the +// sender computed as it sent the data, which the storage node checks against +// its own at `/blob/accept`. +type PutOK struct { + Blob *PutBlob `cborgen:"blob,omitempty" dagjsongen:"blob,omitempty"` +} + +// PutBlob is the blob a `/http/put` delivered. +type PutBlob struct { + Digest multihash.Multihash `cborgen:"digest" dagjsongen:"digest"` +}