fix: expect continue size cap

s3api/middlewares/authentication.go

@@ -28,9 +28,8 @@ import (
 )
 
 const (
-	iso8601Format   = "20060102T150405Z"
-	maxObjSizeLimit = 5 * 1024 * 1024 * 1024 // 5gb
-	defaultRegion   = "us-east-1"
+	iso8601Format = "20060102T150405Z"
+	defaultRegion = "us-east-1"
 )
 
 type RootUserConfig struct {
@@ -178,8 +177,8 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string,
 			}
 			// the upload limit for big data actions: PutObject, UploadPart
 			// is 5gb. If the size exceeds the limit, return 'EntityTooLarge' err
-			if contentLength > maxObjSizeLimit {
-				return s3err.GetEntityTooLargeErr(contentLength, maxObjSizeLimit)
+			if contentLength > utils.MaxObjSizeLimit {
+				return s3err.GetEntityTooLargeErr(contentLength, utils.MaxObjSizeLimit)
 			}
 
 			return nil

s3api/middlewares/presign-auth.go

@@ -90,8 +90,8 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, region
 			}
 			// the upload limit for big data actions: PutObject, UploadPart
 			// is 5gb. If the size exceeds the limit, return 'EntityTooLarge' err
-			if contentLength > maxObjSizeLimit {
-				return s3err.GetEntityTooLargeErr(contentLength, maxObjSizeLimit)
+			if contentLength > utils.MaxObjSizeLimit {
+				return s3err.GetEntityTooLargeErr(contentLength, utils.MaxObjSizeLimit)
 			}
 		}
 

s3api/server.go

@@ -116,6 +116,7 @@ func New(
 		ReadBufferSize: requestHeaderMaxSize,
 	})
 	installRequestHeaderLimitErrorHandler(app)
+	app.Server().ExpectHandler = expectHandler
 
 	server.app = app
 	server.Router.app = app
@@ -453,6 +454,28 @@ func installRequestHeaderLimitErrorHandler(app *fiber.App) {
 
 // globalErrorHandler catches the errors before reaching to
 // the handlers and any system panics
+// expectHandler answers a request's "Expect: 100-continue" before fasthttp
+// tells the client to send the body. A declared Content-Length over the
+// PutObject/UploadPart cap is rejected here with EntityTooLarge, as S3 does,
+// so the client never starts the upload. Without this the middleware still
+// rejects on the same header, but only after fasthttp has sent 100 Continue
+// and the client has begun streaming: the server then closes a connection
+// the client is still writing to, and whether the client reads the 400 or
+// hits the reset first is a race. Everything else proceeds to the handlers.
+func expectHandler(ctx *fasthttp.RequestCtx) int {
+	n := ctx.Request.Header.ContentLength()
+	if n <= utils.MaxObjSizeLimit {
+		return fasthttp.StatusContinue
+	}
+	requestID, hostID := utils.NewS3RequestID(), utils.NewS3HostID()
+	ctx.Response.Header.Set(utils.HeaderAmzRequestID, requestID)
+	ctx.Response.Header.Set(utils.HeaderAmzID2, hostID)
+	ctx.Response.Header.SetContentType(fiber.MIMEApplicationXML)
+	err := s3err.GetEntityTooLargeErr(int64(n), utils.MaxObjSizeLimit)
+	ctx.Response.SetBody(err.XMLBody(requestID, hostID))
+	return err.StatusCode()
+}
+
 func globalErrorHandler(ctx fiber.Ctx, er error) error {
 	requestID, hostID := utils.EnsureRequestIDs(ctx)
 

s3api/server_test.go

@@ -15,10 +15,14 @@
 package s3api
 
 import (
+	"context"
+	"io"
+	"net"
 	"net/http"
 	"net/http/httptest"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -27,6 +31,7 @@ import (
 	"github.com/fil-forge/versitygw/s3api/middlewares"
 	"github.com/fil-forge/versitygw/s3api/utils"
 	"github.com/gofiber/fiber/v3"
+	"github.com/valyala/fasthttp/fasthttputil"
 )
 
 func newTestS3ApiServer(opts ...Option) (*S3ApiServer, error) {
@@ -241,3 +246,72 @@ func TestCustomMountValidation(t *testing.T) {
 		})
 	}
 }
+
+// countingReader counts the bytes a client actually sends as a request body.
+type countingReader struct {
+	n atomic.Int64
+}
+
+func (r *countingReader) Read(p []byte) (int, error) {
+	r.n.Add(int64(len(p)))
+	return len(p), nil
+}
+
+// TestExpectContinue_RejectsOverCapBeforeBody: an upload declaring more than
+// the 5 GiB cap with "Expect: 100-continue" is answered EntityTooLarge before
+// the client sends a single body byte; one within the cap gets its 100
+// Continue and reaches the handlers. Runs the real fasthttp server over an
+// in-memory listener, since the Expect exchange happens below fiber.
+func TestExpectContinue_RejectsOverCapBeforeBody(t *testing.T) {
+	server, err := newTestS3ApiServer()
+	if err != nil {
+		t.Fatalf("New() error = %v", err)
+	}
+	ln := fasthttputil.NewInmemoryListener()
+	go func() { _ = server.app.Server().Serve(ln) }()
+	t.Cleanup(func() { _ = ln.Close() })
+
+	client := &http.Client{Transport: &http.Transport{
+		DialContext:           func(context.Context, string, string) (net.Conn, error) { return ln.Dial() },
+		ExpectContinueTimeout: 5 * time.Second,
+	}}
+	send := func(size int64) (*http.Response, *countingReader) {
+		body := &countingReader{}
+		req, err := http.NewRequest(http.MethodPut, "http://vgw/bucket/key", io.LimitReader(body, size))
+		if err != nil {
+			t.Fatalf("NewRequest: %v", err)
+		}
+		req.ContentLength = size
+		req.Header.Set("Expect", "100-continue")
+		resp, err := client.Do(req)
+		if err != nil {
+			t.Fatalf("PUT of %d bytes: %v", size, err)
+		}
+		return resp, body
+	}
+
+	resp, body := send(utils.MaxObjSizeLimit + 1)
+	xmlBody, _ := io.ReadAll(resp.Body)
+	_ = resp.Body.Close()
+	if resp.StatusCode != http.StatusBadRequest {
+		t.Fatalf("over-cap status = %d, want 400", resp.StatusCode)
+	}
+	if !strings.Contains(string(xmlBody), "<Code>EntityTooLarge</Code>") {
+		t.Fatalf("over-cap body = %q, want an EntityTooLarge error document", xmlBody)
+	}
+	if resp.Header.Get(utils.HeaderAmzRequestID) == "" {
+		t.Fatalf("over-cap response lacks %s", utils.HeaderAmzRequestID)
+	}
+	if got := body.n.Load(); got != 0 {
+		t.Fatalf("client sent %d body bytes before the rejection, want 0", got)
+	}
+
+	// Within the cap the Expect handler steps aside: the request reaches the
+	// S3 handlers, which reject the unsigned request as they would any other.
+	resp, _ = send(1024)
+	within, _ := io.ReadAll(resp.Body)
+	_ = resp.Body.Close()
+	if strings.Contains(string(within), "EntityTooLarge") || resp.StatusCode == http.StatusExpectationFailed {
+		t.Fatalf("within-cap request was refused at the Expect stage: status %d body %q", resp.StatusCode, within)
+	}
+}

s3api/utils/chunk-reader.go

@@ -29,7 +29,12 @@ import (
 )
 
 const (
-	maxObjSizeLimit = 5 * 1024 * 1024 * 1024 // 5gb
+	// MaxObjSizeLimit is the largest single PutObject/UploadPart payload S3
+	// accepts (5 GiB). Enforced on the declared length before the body is
+	// read: by the server's Expect handler for clients that send
+	// "Expect: 100-continue", and by the auth middlewares and the chunk
+	// reader for everything else.
+	MaxObjSizeLimit = 5 * 1024 * 1024 * 1024
 )
 
 type payloadType string
@@ -184,9 +189,9 @@ func ParseDecodedContentLength(ctx fiber.Ctx) (int64, error) {
 		return 0, s3err.GetAPIError(s3err.ErrMissingContentLength)
 	}
 
-	if decContLength > maxObjSizeLimit {
-		debuglogger.Logf("the object size exceeds the allowed limit: (size): %v, (limit): %v", decContLength, int64(maxObjSizeLimit))
-		return 0, s3err.GetEntityTooLargeErr(decContLength, maxObjSizeLimit)
+	if decContLength > MaxObjSizeLimit {
+		debuglogger.Logf("the object size exceeds the allowed limit: (size): %v, (limit): %v", decContLength, int64(MaxObjSizeLimit))
+		return 0, s3err.GetEntityTooLargeErr(decContLength, MaxObjSizeLimit)
 	}
 
 	return decContLength, nil