From 0c8d908dc4cb9020ea2646f8d9ad2536ad7bb199 Mon Sep 17 00:00:00 2001 From: John Murphy Date: Tue, 13 Jan 2026 09:00:42 +1100 Subject: [PATCH 1/9] [WIP] feat: Add s3 cache backend --- go.mod | 18 ++- go.sum | 34 +++++ internal/cache/s3.go | 284 ++++++++++++++++++++++++++++++++++++++ internal/cache/s3_test.go | 98 +++++++++++++ 4 files changed, 433 insertions(+), 1 deletion(-) create mode 100644 internal/cache/s3.go create mode 100644 internal/cache/s3_test.go diff --git a/go.mod b/go.mod index 1a5e865c..c026479d 100644 --- a/go.mod +++ b/go.mod @@ -10,8 +10,24 @@ require ( ) require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/hexops/gotextdiff v1.0.3 // indirect - golang.org/x/sys v0.29.0 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/minio/crc64nvme v1.1.0 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/minio-go/v7 v7.0.97 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/tinylib/msgp v1.3.0 // indirect + golang.org/x/crypto v0.36.0 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/sys v0.34.0 // indirect + golang.org/x/text v0.26.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( diff --git a/go.sum b/go.sum index 6a8c87d2..fbfef50c 100644 --- a/go.sum +++ b/go.sum @@ -12,19 +12,53 @@ github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/lmittmann/tint v1.1.2 h1:2CQzrL6rslrsyjqLDwD11bZ5OpLBPU+g3G/r5LSfS8w= github.com/lmittmann/tint v1.1.2/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE= +github.com/minio/crc64nvme v1.1.0 h1:e/tAguZ+4cw32D+IO/8GSf5UVr9y+3eJcxZI2WOO/7Q= +github.com/minio/crc64nvme v1.1.0/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.97 h1:lqhREPyfgHTB/ciX8k2r8k0D93WaFqxbJX36UZq5occ= +github.com/minio/minio-go/v7 v7.0.97/go.mod h1:re5VXuo0pwEtoNLsNuSr0RrLfT/MBtohwdaSmPPSRSk= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= +github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= +golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/cache/s3.go b/internal/cache/s3.go new file mode 100644 index 00000000..f25ddce1 --- /dev/null +++ b/internal/cache/s3.go @@ -0,0 +1,284 @@ +package cache + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/textproto" + "os" + "time" + + "github.com/alecthomas/errors" + "github.com/alecthomas/kong" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + + "github.com/block/sfptc/internal/logging" +) + +func init() { + Register("s3", NewS3) +} + +type S3Config struct { + Endpoint string `hcl:"endpoint" help:"S3 endpoint URL (e.g., s3.amazonaws.com or localhost:9000)."` + AccessKeyID string `hcl:"access-key-id,optional" help:"S3 access key ID (optional, uses AWS credential chain if not provided)."` + SecretAccessKey string `hcl:"secret-access-key,optional" help:"S3 secret access key (optional, uses AWS credential chain if not provided)."` + Bucket string `hcl:"bucket" help:"S3 bucket name."` + Region string `hcl:"region,optional" help:"S3 region (defaults to us-west-2)." default:"us-west-2"` + UseSSL bool `hcl:"use-ssl,optional" help:"Use SSL for S3 connections (defaults to true)." default:"true"` + MaxTTL time.Duration `hcl:"max-ttl,optional" help:"Maximum time-to-live for entries in the S3 cache (defaults to 1 hour)." default:"1h"` +} + +type S3 struct { + logger *slog.Logger + config S3Config + client *minio.Client +} + +var _ Cache = (*S3)(nil) + +// credentialMode returns a string indicating which credential mode is being used. +func credentialMode(config S3Config) string { + if config.AccessKeyID != "" && config.SecretAccessKey != "" { + return "static" + } + return "aws-chain" +} + +// NewS3 creates a new S3-based cache instance using the minio SDK. +// +// config.Endpoint and config.Bucket MUST be set. +// +// If config.AccessKeyID and config.SecretAccessKey are provided, static credentials will be used. +// Otherwise, the standard AWS credential chain will be used, which includes: +// 1. Environment variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN) +// 2. AWS credentials file (~/.aws/credentials) +// 3. IAM role from EC2 instance metadata or ECS container credentials +// +// This [Cache] implementation stores cache entries in an S3-compatible object storage service. +// Metadata (headers and expiration time) are stored as object user metadata. The implementation +// uses the lightweight minio-go SDK to reduce overhead compared to the AWS SDK. +func NewS3(ctx context.Context, config S3Config) (*S3, error) { + logging.FromContext(ctx).InfoContext(ctx, "Constructing S3 cache", + "endpoint", config.Endpoint, + "bucket", config.Bucket, + "region", config.Region, + "use-ssl", config.UseSSL, + "max-ttl", config.MaxTTL, + "credentials-mode", credentialMode(config)) + + // Validate config + if config.Endpoint == "" { + return nil, errors.New("endpoint is required") + } + if config.Bucket == "" { + return nil, errors.New("bucket is required") + } + + err := kong.ApplyDefaults(&config) + if err != nil { + return nil, errors.Errorf("failed to apply defaults: %w", err) + } + + // Determine credential provider + var creds *credentials.Credentials + if config.AccessKeyID != "" && config.SecretAccessKey != "" { + // Use static credentials if both are provided + creds = credentials.NewStaticV4(config.AccessKeyID, config.SecretAccessKey, "") + } else if config.AccessKeyID != "" || config.SecretAccessKey != "" { + // Error if only one is provided + return nil, errors.New("both access-key-id and secret-access-key must be provided together, or neither for credential chain") + } else { + // Use AWS credential chain if neither is provided + transport, err := minio.DefaultTransport(config.UseSSL) + if err != nil { + return nil, errors.Errorf("failed to create default transport: %w", err) + } + creds = credentials.NewChainCredentials( + []credentials.Provider{ + &credentials.EnvAWS{}, // Check AWS environment variables + &credentials.FileAWSCredentials{}, // Check ~/.aws/credentials + &credentials.IAM{ + Client: &http.Client{ + Transport: transport, + }, + }, // Check EC2 instance metadata or ECS container credentials + }) + } + + // Create minio client + client, err := minio.New(config.Endpoint, &minio.Options{ + Creds: creds, + Secure: config.UseSSL, + Region: config.Region, + }) + if err != nil { + return nil, errors.Errorf("failed to create minio client: %w", err) + } + + // Verify bucket exists + exists, err := client.BucketExists(ctx, config.Bucket) + if err != nil { + return nil, errors.Errorf("failed to check if bucket exists: %w", err) + } + if !exists { + return nil, errors.Errorf("bucket %s does not exist", config.Bucket) + } + + return &S3{ + logger: logging.FromContext(ctx), + config: config, + client: client, + }, nil +} + +func (s *S3) String() string { + return fmt.Sprintf("s3:%s/%s", s.config.Endpoint, s.config.Bucket) +} + +func (s *S3) Close() error { + return nil +} + +func (s *S3) Open(ctx context.Context, key Key) (io.ReadCloser, textproto.MIMEHeader, error) { + objectName := key.String() + + // Get object info to check metadata + objInfo, err := s.client.StatObject(ctx, s.config.Bucket, objectName, minio.StatObjectOptions{}) + if err != nil { + errResponse := minio.ToErrorResponse(err) + if errResponse.Code == "NoSuchKey" { + return nil, nil, os.ErrNotExist + } + return nil, nil, errors.Errorf("failed to stat object: %w", err) + } + + // Check if object has expired + expiresAtStr := objInfo.UserMetadata["X-Amz-Meta-Expires-At"] + if expiresAtStr != "" { + var expiresAt time.Time + if err := expiresAt.UnmarshalText([]byte(expiresAtStr)); err == nil { + if time.Now().After(expiresAt) { + // Object expired, delete it and return not found + return nil, nil, errors.Join(os.ErrNotExist, s.Delete(ctx, key)) + } + } + } + + // Retrieve headers from metadata + headers := make(textproto.MIMEHeader) + if headersJSON := objInfo.UserMetadata["X-Amz-Meta-Headers"]; headersJSON != "" { + if err := json.Unmarshal([]byte(headersJSON), &headers); err != nil { + return nil, nil, errors.Errorf("failed to unmarshal headers: %w", err) + } + } + + // Get object + obj, err := s.client.GetObject(ctx, s.config.Bucket, objectName, minio.GetObjectOptions{}) + if err != nil { + return nil, nil, errors.Errorf("failed to get object: %w", err) + } + + return obj, headers, nil +} + +func (s *S3) Create(ctx context.Context, key Key, headers textproto.MIMEHeader, ttl time.Duration) (io.WriteCloser, error) { + if ttl > s.config.MaxTTL || ttl == 0 { + ttl = s.config.MaxTTL + } + + expiresAt := time.Now().Add(ttl) + + return &s3Writer{ + s3: s, + key: key, + buf: &bytes.Buffer{}, + expiresAt: expiresAt, + headers: headers, + ctx: ctx, + }, nil +} + +func (s *S3) Delete(ctx context.Context, key Key) error { + objectName := key.String() + + // Check if object exists first + _, err := s.client.StatObject(ctx, s.config.Bucket, objectName, minio.StatObjectOptions{}) + if err != nil { + errResponse := minio.ToErrorResponse(err) + if errResponse.Code == "NoSuchKey" { + return os.ErrNotExist + } + return errors.Errorf("failed to stat object: %w", err) + } + + err = s.client.RemoveObject(ctx, s.config.Bucket, objectName, minio.RemoveObjectOptions{}) + if err != nil { + return errors.Errorf("failed to remove object: %w", err) + } + + return nil +} + +type s3Writer struct { + s3 *S3 + key Key + buf *bytes.Buffer + expiresAt time.Time + headers textproto.MIMEHeader + ctx context.Context +} + +func (w *s3Writer) Write(p []byte) (int, error) { + return errors.WithStack2(w.buf.Write(p)) +} + +func (w *s3Writer) Close() error { + // Check if context was cancelled + if err := w.ctx.Err(); err != nil { + return errors.Wrap(err, "create operation cancelled") + } + + objectName := w.key.String() + + // Prepare user metadata + userMetadata := make(map[string]string) + + // Store expiration time + expiresAtBytes, err := w.expiresAt.MarshalText() + if err != nil { + return errors.Errorf("failed to marshal expiration time: %w", err) + } + userMetadata["Expires-At"] = string(expiresAtBytes) + + // Store headers as JSON + if len(w.headers) > 0 { + headersJSON, err := json.Marshal(w.headers) + if err != nil { + return errors.Errorf("failed to marshal headers: %w", err) + } + userMetadata["Headers"] = string(headersJSON) + } + + // Upload object + _, err = w.s3.client.PutObject( + w.ctx, + w.s3.config.Bucket, + objectName, + w.buf, + int64(w.buf.Len()), + minio.PutObjectOptions{ + UserMetadata: userMetadata, + }, + ) + if err != nil { + return errors.Errorf("failed to put object: %w", err) + } + + return nil +} diff --git a/internal/cache/s3_test.go b/internal/cache/s3_test.go new file mode 100644 index 00000000..20c59f47 --- /dev/null +++ b/internal/cache/s3_test.go @@ -0,0 +1,98 @@ +package cache_test + +import ( + "log/slog" + "os" + "testing" + "time" + + "github.com/alecthomas/assert/v2" + + "github.com/block/sfptc/internal/cache" + "github.com/block/sfptc/internal/cache/cachetest" + "github.com/block/sfptc/internal/logging" +) + +// TestS3Cache tests the S3 cache implementation. +// +// This test requires the following environment variables to be set: +// - S3_TEST_ENDPOINT: S3 endpoint (e.g., localhost:9000 for local minio, or s3.amazonaws.com for AWS) +// - S3_TEST_BUCKET: S3 bucket name +// - S3_TEST_REGION: S3 region (optional, defaults to us-east-1) +// - S3_TEST_USE_SSL: Whether to use SSL (optional, defaults to true) +// +// For credentials, you have two options: +// +// Option 1: Explicit credentials (required for local minio): +// - S3_TEST_ACCESS_KEY_ID: S3 access key ID +// - S3_TEST_SECRET_ACCESS_KEY: S3 secret access key +// +// Option 2: AWS credential chain (for AWS S3 or EC2 instances with IAM roles): +// - Leave S3_TEST_ACCESS_KEY_ID and S3_TEST_SECRET_ACCESS_KEY unset +// - The test will use the standard AWS credential chain: +// 1. AWS environment variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) +// 2. AWS credentials file (~/.aws/credentials) +// 3. IAM role from EC2 instance metadata +// +// To run tests against a local minio server: +// +// docker run -d -p 9000:9000 -p 9001:9001 \ +// -e "MINIO_ROOT_USER=minioadmin" \ +// -e "MINIO_ROOT_PASSWORD=minioadmin" \ +// minio/minio server /data --console-address ":9001" +// +// # Create a test bucket using mc (minio client) +// docker exec mc alias set local http://localhost:9000 minioadmin minioadmin +// docker exec mc mb local/test-bucket +// +// export S3_TEST_ENDPOINT=localhost:9000 +// export S3_TEST_ACCESS_KEY_ID=minioadmin +// export S3_TEST_SECRET_ACCESS_KEY=minioadmin +// export S3_TEST_BUCKET=test-bucket +// export S3_TEST_USE_SSL=false +// go test -v ./internal/cache -run TestS3Cache +// +// To run tests against AWS S3 with IAM credentials: +// +// export S3_TEST_ENDPOINT=s3.amazonaws.com +// export S3_TEST_BUCKET=my-test-bucket +// export S3_TEST_REGION=us-east-1 +// # AWS credentials will be picked up from environment, credentials file, or IAM role +// go test -v ./internal/cache -run TestS3Cache +func TestS3Cache(t *testing.T) { + endpoint := os.Getenv("S3_TEST_ENDPOINT") + bucket := os.Getenv("S3_TEST_BUCKET") + + if endpoint == "" || bucket == "" { + t.Skip("Skipping S3 cache tests: S3_TEST_ENDPOINT and S3_TEST_BUCKET environment variables must be set") + } + + // Credentials are optional - will use AWS credential chain if not provided + accessKeyID := os.Getenv("S3_TEST_ACCESS_KEY_ID") + secretAccessKey := os.Getenv("S3_TEST_SECRET_ACCESS_KEY") + + region := os.Getenv("S3_TEST_REGION") + if region == "" { + region = "us-west-2" + } + + useSSL := true + if os.Getenv("S3_TEST_USE_SSL") == "false" { + useSSL = false + } + + cachetest.Suite(t, func(t *testing.T) cache.Cache { + _, ctx := logging.Configure(t.Context(), logging.Config{Level: slog.LevelDebug}) + c, err := cache.NewS3(ctx, cache.S3Config{ + Endpoint: endpoint, + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + Bucket: bucket, + Region: region, + UseSSL: useSSL, + MaxTTL: 100 * time.Millisecond, + }) + assert.NoError(t, err) + return c + }) +} From 7f89888c45dee3e74a9492bf05cf4a891b4ec83c Mon Sep 17 00:00:00 2001 From: John Murphy Date: Tue, 13 Jan 2026 10:40:17 +1100 Subject: [PATCH 2/9] fix: issue talking to minio test container --- go.mod | 61 ++++++++++++- go.sum | 169 +++++++++++++++++++++++++++++++--- internal/cache/s3.go | 81 +++++++++++----- internal/cache/s3_test.go | 188 +++++++++++++++++++++++++------------- 4 files changed, 392 insertions(+), 107 deletions(-) diff --git a/go.mod b/go.mod index c026479d..65720eb8 100644 --- a/go.mod +++ b/go.mod @@ -6,27 +6,78 @@ require ( github.com/alecthomas/hcl/v2 v2.3.1 github.com/alecthomas/kong v1.13.0 github.com/lmittmann/tint v1.1.2 + github.com/minio/minio-go/v7 v7.0.97 + github.com/testcontainers/testcontainers-go/modules/minio v0.40.0 go.etcd.io/bbolt v1.4.3 ) require ( + dario.cat/mergo v1.0.2 // indirect + github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect + github.com/cpuguy83/dockercfg v0.3.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/docker v28.5.1+incompatible // indirect + github.com/docker/go-connections v0.6.0 // indirect + github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/ebitengine/purego v0.8.4 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-ini/ini v1.67.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.4 // indirect github.com/hexops/gotextdiff v1.0.3 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.2.11 // indirect github.com/klauspost/crc32 v1.3.0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/magiconair/properties v1.8.10 // indirect github.com/minio/crc64nvme v1.1.0 // indirect github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/minio-go/v7 v7.0.97 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/go-archive v0.1.0 // indirect + github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/sys/sequential v0.6.0 // indirect + github.com/moby/sys/user v0.4.0 // indirect + github.com/moby/sys/userns v0.1.0 // indirect + github.com/moby/term v0.5.0 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect github.com/philhofer/fwd v1.2.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/rs/xid v1.6.0 // indirect + github.com/shirou/gopsutil/v4 v4.25.6 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/stretchr/testify v1.11.1 // indirect + github.com/testcontainers/testcontainers-go v0.40.0 // indirect github.com/tinylib/msgp v1.3.0 // indirect - golang.org/x/crypto v0.36.0 // indirect - golang.org/x/net v0.38.0 // indirect - golang.org/x/sys v0.34.0 // indirect - golang.org/x/text v0.26.0 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.38.0 // indirect + go.opentelemetry.io/otel/metric v1.38.0 // indirect + go.opentelemetry.io/otel/sdk v1.38.0 // indirect + go.opentelemetry.io/otel/trace v1.38.0 // indirect + golang.org/x/crypto v0.44.0 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/text v0.32.0 // indirect + google.golang.org/grpc v1.78.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index fbfef50c..9ec6eac5 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,11 @@ +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/errors v0.9.1 h1:JNXtU30rtMNARCkW41OTZ4yL6Lyocq20xIJgIw2raqI= @@ -10,14 +18,53 @@ github.com/alecthomas/participle/v2 v2.1.4 h1:W/H79S8Sat/krZ3el6sQMvMaahJ+XcM9WS github.com/alecthomas/participle/v2 v2.1.4/go.mod h1:8tqVbpTX20Ru4NfYQgZf4mP18eXPTBViyMWiArNEgGI= github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs= github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v28.5.1+incompatible h1:Bm8DchhSD2J6PsFzxC35TZo4TLGR2PdW/E69rU45NhM= +github.com/docker/docker v28.5.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= +github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw= +github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.4 h1:kEISI/Gx67NzH3nJxAmY/dGac80kKZgZt134u7Y/k1s= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.4/go.mod h1:6Nz966r3vQYCqIzWsuEl9d7cf7mRhtDmm++sOxlnfxI= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= @@ -27,38 +74,132 @@ github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4O github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lmittmann/tint v1.1.2 h1:2CQzrL6rslrsyjqLDwD11bZ5OpLBPU+g3G/r5LSfS8w= github.com/lmittmann/tint v1.1.2/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= +github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/minio/crc64nvme v1.1.0 h1:e/tAguZ+4cw32D+IO/8GSf5UVr9y+3eJcxZI2WOO/7Q= github.com/minio/crc64nvme v1.1.0/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.97 h1:lqhREPyfgHTB/ciX8k2r8k0D93WaFqxbJX36UZq5occ= github.com/minio/minio-go/v7 v7.0.97/go.mod h1:re5VXuo0pwEtoNLsNuSr0RrLfT/MBtohwdaSmPPSRSk= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/go-archive v0.1.0 h1:Kk/5rdW/g+H8NHdJW2gsXyZ7UnzvJNOy6VKJqueWdcQ= +github.com/moby/go-archive v0.1.0/go.mod h1:G9B+YoujNohJmrIYFBpSd54GTUB4lt9S+xVQvsJyFuo= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= +github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= +github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= +github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= +github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/shirou/gopsutil/v4 v4.25.6 h1:kLysI2JsKorfaFPcYmcJqbzROzsBWEOAtw6A7dIfqXs= +github.com/shirou/gopsutil/v4 v4.25.6/go.mod h1:PfybzyydfZcN+JMMjkF6Zb8Mq1A/VcogFFg7hj50W9c= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/testcontainers/testcontainers-go v0.40.0 h1:pSdJYLOVgLE8YdUY2FHQ1Fxu+aMnb6JfVz1mxk7OeMU= +github.com/testcontainers/testcontainers-go v0.40.0/go.mod h1:FSXV5KQtX2HAMlm7U3APNyLkkap35zNLxukw9oBi/MY= +github.com/testcontainers/testcontainers-go/modules/minio v0.40.0 h1:M+Ib1mIXq/hEcH8tyEvBnOZ7NJi03zY+P1gYO5GGp6o= +github.com/testcontainers/testcontainers-go/modules/minio v0.40.0/go.mod h1:ON0MxxS/pME0SJOKLImw/D9R1L7apYsxIZrM/uEqORA= github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= -golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= -golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= -golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= +golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU= +golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= +golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/api v0.0.0-20251222181119-0a764e51fe1b h1:uA40e2M6fYRBf0+8uN5mLlqUtV192iiksiICIBkYJ1E= +google.golang.org/genproto/googleapis/api v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:Xa7le7qx2vmqB/SzWUBa7KdMjpdpAHlh5QCSnjessQk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1:Mv8VFug0MP9e5vUxfBcE3vUkV6CImK3cMNMIDFjmzxU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= +google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= diff --git a/internal/cache/s3.go b/internal/cache/s3.go index f25ddce1..6c4e5a92 100644 --- a/internal/cache/s3.go +++ b/internal/cache/s3.go @@ -3,6 +3,7 @@ package cache import ( "bytes" "context" + "crypto/tls" "encoding/json" "fmt" "io" @@ -13,7 +14,6 @@ import ( "time" "github.com/alecthomas/errors" - "github.com/alecthomas/kong" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" @@ -29,9 +29,10 @@ type S3Config struct { AccessKeyID string `hcl:"access-key-id,optional" help:"S3 access key ID (optional, uses AWS credential chain if not provided)."` SecretAccessKey string `hcl:"secret-access-key,optional" help:"S3 secret access key (optional, uses AWS credential chain if not provided)."` Bucket string `hcl:"bucket" help:"S3 bucket name."` - Region string `hcl:"region,optional" help:"S3 region (defaults to us-west-2)." default:"us-west-2"` - UseSSL bool `hcl:"use-ssl,optional" help:"Use SSL for S3 connections (defaults to true)." default:"true"` - MaxTTL time.Duration `hcl:"max-ttl,optional" help:"Maximum time-to-live for entries in the S3 cache (defaults to 1 hour)." default:"1h"` + Region string `hcl:"region,optional" help:"S3 region (defaults to us-west-2)."` + UseSSL *bool `hcl:"use-ssl,optional" help:"Use SSL for S3 connections (defaults to true)."` + SkipSSLVerify bool `hcl:"skip-ssl-verify,optional" help:"Skip SSL certificate verification (defaults to false)."` + MaxTTL time.Duration `hcl:"max-ttl,optional" help:"Maximum time-to-live for entries in the S3 cache (defaults to 1 hour)."` } type S3 struct { @@ -64,14 +65,6 @@ func credentialMode(config S3Config) string { // Metadata (headers and expiration time) are stored as object user metadata. The implementation // uses the lightweight minio-go SDK to reduce overhead compared to the AWS SDK. func NewS3(ctx context.Context, config S3Config) (*S3, error) { - logging.FromContext(ctx).InfoContext(ctx, "Constructing S3 cache", - "endpoint", config.Endpoint, - "bucket", config.Bucket, - "region", config.Region, - "use-ssl", config.UseSSL, - "max-ttl", config.MaxTTL, - "credentials-mode", credentialMode(config)) - // Validate config if config.Endpoint == "" { return nil, errors.New("endpoint is required") @@ -80,13 +73,46 @@ func NewS3(ctx context.Context, config S3Config) (*S3, error) { return nil, errors.New("bucket is required") } - err := kong.ApplyDefaults(&config) - if err != nil { - return nil, errors.Errorf("failed to apply defaults: %w", err) + // Apply defaults only for zero values + if config.Region == "" { + config.Region = "us-west-2" + } + if config.MaxTTL == 0 { + config.MaxTTL = time.Hour + } + // UseSSL defaults to true if not explicitly set + useSSL := true + if config.UseSSL != nil { + useSSL = *config.UseSSL } - // Determine credential provider + logging.FromContext(ctx).InfoContext(ctx, "Constructing S3 cache", + "endpoint", config.Endpoint, + "bucket", config.Bucket, + "region", config.Region, + "use-ssl", useSSL, + "max-ttl", config.MaxTTL, + "credentials-mode", credentialMode(config)) + + // Determine credential provider and optional custom transport var creds *credentials.Credentials + var transport http.RoundTripper + + // Only create custom transport if we need to skip SSL verification + if config.SkipSSLVerify { + defaultTransport, err := minio.DefaultTransport(useSSL) + if err != nil { + return nil, errors.Errorf("failed to create default transport: %w", err) + } + // Clone the default transport and disable SSL verification + customTransport := defaultTransport.Clone() + if customTransport.TLSClientConfig == nil { + customTransport.TLSClientConfig = &tls.Config{} + } + customTransport.TLSClientConfig.InsecureSkipVerify = true + transport = customTransport + } + if config.AccessKeyID != "" && config.SecretAccessKey != "" { // Use static credentials if both are provided creds = credentials.NewStaticV4(config.AccessKeyID, config.SecretAccessKey, "") @@ -95,28 +121,39 @@ func NewS3(ctx context.Context, config S3Config) (*S3, error) { return nil, errors.New("both access-key-id and secret-access-key must be provided together, or neither for credential chain") } else { // Use AWS credential chain if neither is provided - transport, err := minio.DefaultTransport(config.UseSSL) + defaultTransport, err := minio.DefaultTransport(useSSL) if err != nil { return nil, errors.Errorf("failed to create default transport: %w", err) } + if transport != nil { + // Use custom transport if already set (for SkipSSLVerify) + defaultTransport = transport.(*http.Transport) + } creds = credentials.NewChainCredentials( []credentials.Provider{ &credentials.EnvAWS{}, // Check AWS environment variables &credentials.FileAWSCredentials{}, // Check ~/.aws/credentials &credentials.IAM{ Client: &http.Client{ - Transport: transport, + Transport: defaultTransport, }, }, // Check EC2 instance metadata or ECS container credentials }) } - // Create minio client - client, err := minio.New(config.Endpoint, &minio.Options{ + // Create minio client options + options := &minio.Options{ Creds: creds, - Secure: config.UseSSL, + Secure: useSSL, Region: config.Region, - }) + } + + // Only set custom transport if needed (for SkipSSLVerify) + if transport != nil { + options.Transport = transport + } + + client, err := minio.New(config.Endpoint, options) if err != nil { return nil, errors.Errorf("failed to create minio client: %w", err) } diff --git a/internal/cache/s3_test.go b/internal/cache/s3_test.go index 20c59f47..d190f1b8 100644 --- a/internal/cache/s3_test.go +++ b/internal/cache/s3_test.go @@ -1,95 +1,151 @@ package cache_test import ( + "context" + "fmt" "log/slog" + "net/url" "os" "testing" "time" "github.com/alecthomas/assert/v2" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + testcontainersminio "github.com/testcontainers/testcontainers-go/modules/minio" "github.com/block/sfptc/internal/cache" "github.com/block/sfptc/internal/cache/cachetest" "github.com/block/sfptc/internal/logging" ) -// TestS3Cache tests the S3 cache implementation. -// -// This test requires the following environment variables to be set: -// - S3_TEST_ENDPOINT: S3 endpoint (e.g., localhost:9000 for local minio, or s3.amazonaws.com for AWS) -// - S3_TEST_BUCKET: S3 bucket name -// - S3_TEST_REGION: S3 region (optional, defaults to us-east-1) -// - S3_TEST_USE_SSL: Whether to use SSL (optional, defaults to true) -// -// For credentials, you have two options: -// -// Option 1: Explicit credentials (required for local minio): -// - S3_TEST_ACCESS_KEY_ID: S3 access key ID -// - S3_TEST_SECRET_ACCESS_KEY: S3 secret access key -// -// Option 2: AWS credential chain (for AWS S3 or EC2 instances with IAM roles): -// - Leave S3_TEST_ACCESS_KEY_ID and S3_TEST_SECRET_ACCESS_KEY unset -// - The test will use the standard AWS credential chain: -// 1. AWS environment variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) -// 2. AWS credentials file (~/.aws/credentials) -// 3. IAM role from EC2 instance metadata -// -// To run tests against a local minio server: -// -// docker run -d -p 9000:9000 -p 9001:9001 \ -// -e "MINIO_ROOT_USER=minioadmin" \ -// -e "MINIO_ROOT_PASSWORD=minioadmin" \ -// minio/minio server /data --console-address ":9001" -// -// # Create a test bucket using mc (minio client) -// docker exec mc alias set local http://localhost:9000 minioadmin minioadmin -// docker exec mc mb local/test-bucket -// -// export S3_TEST_ENDPOINT=localhost:9000 -// export S3_TEST_ACCESS_KEY_ID=minioadmin -// export S3_TEST_SECRET_ACCESS_KEY=minioadmin -// export S3_TEST_BUCKET=test-bucket -// export S3_TEST_USE_SSL=false -// go test -v ./internal/cache -run TestS3Cache -// -// To run tests against AWS S3 with IAM credentials: -// -// export S3_TEST_ENDPOINT=s3.amazonaws.com -// export S3_TEST_BUCKET=my-test-bucket -// export S3_TEST_REGION=us-east-1 -// # AWS credentials will be picked up from environment, credentials file, or IAM role -// go test -v ./internal/cache -run TestS3Cache -func TestS3Cache(t *testing.T) { - endpoint := os.Getenv("S3_TEST_ENDPOINT") - bucket := os.Getenv("S3_TEST_BUCKET") +var ( + minioContainer *testcontainersminio.MinioContainer + minioEndpoint string + minioBucket = "test-bucket" + minioUsername = "minioadmin" + minioPassword = "minioadmin" +) + +// TestMain manages the MinIO container lifecycle for the entire test package. +// The container is started once before all tests run and terminated after all tests complete. +func TestMain(m *testing.M) { + ctx := context.Background() - if endpoint == "" || bucket == "" { - t.Skip("Skipping S3 cache tests: S3_TEST_ENDPOINT and S3_TEST_BUCKET environment variables must be set") + // Check for opt-out environment variable + if os.Getenv("SKIP_TESTCONTAINERS") != "" { + fmt.Println("Skipping testcontainers setup (SKIP_TESTCONTAINERS is set)") + os.Exit(m.Run()) } - // Credentials are optional - will use AWS credential chain if not provided - accessKeyID := os.Getenv("S3_TEST_ACCESS_KEY_ID") - secretAccessKey := os.Getenv("S3_TEST_SECRET_ACCESS_KEY") + // Start MinIO container + var err error + minioContainer, err = testcontainersminio.Run(ctx, + "minio/minio:RELEASE.2024-01-16T16-07-38Z", + testcontainersminio.WithUsername(minioUsername), + testcontainersminio.WithPassword(minioPassword), + ) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to start MinIO container: %v\n", err) + fmt.Fprintf(os.Stderr, "Ensure Docker is running and accessible.\n") + os.Exit(1) + } - region := os.Getenv("S3_TEST_REGION") - if region == "" { - region = "us-west-2" + // Get connection details + connStr, err := minioContainer.ConnectionString(ctx) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to get MinIO connection string: %v\n", err) + _ = minioContainer.Terminate(ctx) + os.Exit(1) } - useSSL := true - if os.Getenv("S3_TEST_USE_SSL") == "false" { - useSSL = false + // ConnectionString returns just "host:port", but we need to handle it properly + // The minio-go SDK expects just the host:port without protocol + parsedURL, err := url.Parse(connStr) + if err != nil { + // If it can't be parsed as URL, it might already be just host:port + minioEndpoint = connStr + } else if parsedURL.Host != "" { + // If it parsed successfully and has a Host, use that + minioEndpoint = parsedURL.Host + } else { + // Otherwise use the raw string + minioEndpoint = connStr + } + + // Create test bucket + if err := createBucket(ctx); err != nil { + fmt.Fprintf(os.Stderr, "Failed to create test bucket: %v\n", err) + _ = minioContainer.Terminate(ctx) + os.Exit(1) + } + + // Run tests + code := m.Run() + + // Cleanup + if err := minioContainer.Terminate(ctx); err != nil { + fmt.Fprintf(os.Stderr, "Failed to terminate MinIO container: %v\n", err) + } + + os.Exit(code) +} + +// createBucket creates the test bucket in the MinIO container. +func createBucket(ctx context.Context) error { + // Use the minio-go SDK (already in dependencies) to create bucket + client, err := minio.New(minioEndpoint, &minio.Options{ + Creds: credentials.NewStaticV4(minioUsername, minioPassword, ""), + Secure: false, + }) + if err != nil { + return fmt.Errorf("failed to create minio client: %w", err) + } + + // Create bucket if it doesn't exist + exists, err := client.BucketExists(ctx, minioBucket) + if err != nil { + return fmt.Errorf("failed to check if bucket exists: %w", err) + } + + if !exists { + if err := client.MakeBucket(ctx, minioBucket, minio.MakeBucketOptions{}); err != nil { + return fmt.Errorf("failed to create bucket: %w", err) + } + } + + return nil +} + +// TestS3Cache tests the S3 cache implementation using testcontainers-go. +// +// This test automatically starts a MinIO container using testcontainers-go. +// Docker must be running for these tests to execute. +// +// To skip these tests (e.g., during development without Docker): +// +// SKIP_TESTCONTAINERS=1 go test ./internal/cache +// +// The MinIO container: +// - Starts once per test package run +// - Uses credentials: minioadmin/minioadmin +// - Listens on a random available port +// - Cleans up automatically after tests complete +func TestS3Cache(t *testing.T) { + if minioContainer == nil { + t.Skip("MinIO container not available - Docker may not be running or SKIP_TESTCONTAINERS is set") } cachetest.Suite(t, func(t *testing.T) cache.Cache { _, ctx := logging.Configure(t.Context(), logging.Config{Level: slog.LevelDebug}) + useSSL := false c, err := cache.NewS3(ctx, cache.S3Config{ - Endpoint: endpoint, - AccessKeyID: accessKeyID, - SecretAccessKey: secretAccessKey, - Bucket: bucket, - Region: region, - UseSSL: useSSL, + Endpoint: minioEndpoint, + AccessKeyID: minioUsername, + SecretAccessKey: minioPassword, + Bucket: minioBucket, + Region: "", + UseSSL: &useSSL, // MinIO container serves HTTP, not HTTPS MaxTTL: 100 * time.Millisecond, }) assert.NoError(t, err) From a7a9c664b8a0d636fc030db5cee7291b7d1c4d3c Mon Sep 17 00:00:00 2001 From: John Murphy Date: Tue, 13 Jan 2026 12:47:08 +1100 Subject: [PATCH 3/9] fix: Linting issues --- internal/cache/s3.go | 19 ++++++++++++++----- internal/cache/s3_test.go | 8 ++++---- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/internal/cache/s3.go b/internal/cache/s3.go index 6c4e5a92..fc697765 100644 --- a/internal/cache/s3.go +++ b/internal/cache/s3.go @@ -107,19 +107,24 @@ func NewS3(ctx context.Context, config S3Config) (*S3, error) { // Clone the default transport and disable SSL verification customTransport := defaultTransport.Clone() if customTransport.TLSClientConfig == nil { - customTransport.TLSClientConfig = &tls.Config{} + customTransport.TLSClientConfig = &tls.Config{ + MinVersion: tls.VersionTLS12, + } + } else { + customTransport.TLSClientConfig.MinVersion = tls.VersionTLS12 } customTransport.TLSClientConfig.InsecureSkipVerify = true transport = customTransport } - if config.AccessKeyID != "" && config.SecretAccessKey != "" { + switch { + case config.AccessKeyID != "" && config.SecretAccessKey != "": // Use static credentials if both are provided creds = credentials.NewStaticV4(config.AccessKeyID, config.SecretAccessKey, "") - } else if config.AccessKeyID != "" || config.SecretAccessKey != "" { + case config.AccessKeyID != "" || config.SecretAccessKey != "": // Error if only one is provided return nil, errors.New("both access-key-id and secret-access-key must be provided together, or neither for credential chain") - } else { + default: // Use AWS credential chain if neither is provided defaultTransport, err := minio.DefaultTransport(useSSL) if err != nil { @@ -127,7 +132,11 @@ func NewS3(ctx context.Context, config S3Config) (*S3, error) { } if transport != nil { // Use custom transport if already set (for SkipSSLVerify) - defaultTransport = transport.(*http.Transport) + var ok bool + defaultTransport, ok = transport.(*http.Transport) + if !ok { + return nil, errors.New("transport is not an *http.Transport") + } } creds = credentials.NewChainCredentials( []credentials.Provider{ diff --git a/internal/cache/s3_test.go b/internal/cache/s3_test.go index d190f1b8..21c24e40 100644 --- a/internal/cache/s3_test.go +++ b/internal/cache/s3_test.go @@ -34,7 +34,6 @@ func TestMain(m *testing.M) { // Check for opt-out environment variable if os.Getenv("SKIP_TESTCONTAINERS") != "" { - fmt.Println("Skipping testcontainers setup (SKIP_TESTCONTAINERS is set)") os.Exit(m.Run()) } @@ -62,13 +61,14 @@ func TestMain(m *testing.M) { // ConnectionString returns just "host:port", but we need to handle it properly // The minio-go SDK expects just the host:port without protocol parsedURL, err := url.Parse(connStr) - if err != nil { + switch { + case err != nil: // If it can't be parsed as URL, it might already be just host:port minioEndpoint = connStr - } else if parsedURL.Host != "" { + case parsedURL.Host != "": // If it parsed successfully and has a Host, use that minioEndpoint = parsedURL.Host - } else { + default: // Otherwise use the raw string minioEndpoint = connStr } From 3a3b166b8ad4b2a78fd4fe76a26baf3f36080bc4 Mon Sep 17 00:00:00 2001 From: John Murphy Date: Tue, 13 Jan 2026 13:03:23 +1100 Subject: [PATCH 4/9] fix: Issue with expires-at implementation --- internal/cache/s3.go | 6 ++++-- internal/cache/s3_test.go | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/internal/cache/s3.go b/internal/cache/s3.go index fc697765..90be3b59 100644 --- a/internal/cache/s3.go +++ b/internal/cache/s3.go @@ -205,7 +205,8 @@ func (s *S3) Open(ctx context.Context, key Key) (io.ReadCloser, textproto.MIMEHe } // Check if object has expired - expiresAtStr := objInfo.UserMetadata["X-Amz-Meta-Expires-At"] + // Note: UserMetadata keys are returned WITHOUT the "X-Amz-Meta-" prefix by minio-go + expiresAtStr := objInfo.UserMetadata["Expires-At"] if expiresAtStr != "" { var expiresAt time.Time if err := expiresAt.UnmarshalText([]byte(expiresAtStr)); err == nil { @@ -217,8 +218,9 @@ func (s *S3) Open(ctx context.Context, key Key) (io.ReadCloser, textproto.MIMEHe } // Retrieve headers from metadata + // Note: UserMetadata keys are returned WITHOUT the "X-Amz-Meta-" prefix by minio-go headers := make(textproto.MIMEHeader) - if headersJSON := objInfo.UserMetadata["X-Amz-Meta-Headers"]; headersJSON != "" { + if headersJSON := objInfo.UserMetadata["Headers"]; headersJSON != "" { if err := json.Unmarshal([]byte(headersJSON), &headers); err != nil { return nil, nil, errors.Errorf("failed to unmarshal headers: %w", err) } diff --git a/internal/cache/s3_test.go b/internal/cache/s3_test.go index 21c24e40..266e7e14 100644 --- a/internal/cache/s3_test.go +++ b/internal/cache/s3_test.go @@ -138,6 +138,28 @@ func TestS3Cache(t *testing.T) { cachetest.Suite(t, func(t *testing.T) cache.Cache { _, ctx := logging.Configure(t.Context(), logging.Config{Level: slog.LevelDebug}) + + // Clean up any existing objects in the bucket before creating a new cache instance + // This ensures test isolation since all tests share the same bucket + client, err := minio.New(minioEndpoint, &minio.Options{ + Creds: credentials.NewStaticV4(minioUsername, minioPassword, ""), + Secure: false, + }) + assert.NoError(t, err) + + // Remove all objects from the bucket + objectsCh := client.ListObjects(ctx, minioBucket, minio.ListObjectsOptions{Recursive: true}) + for obj := range objectsCh { + if obj.Err != nil { + t.Logf("Error listing objects: %v", obj.Err) + continue + } + err := client.RemoveObject(ctx, minioBucket, obj.Key, minio.RemoveObjectOptions{}) + if err != nil { + t.Logf("Error removing object %s: %v", obj.Key, err) + } + } + useSSL := false c, err := cache.NewS3(ctx, cache.S3Config{ Endpoint: minioEndpoint, From 19490369d35eca48917e13a735e7a0994b48614b Mon Sep 17 00:00:00 2001 From: John Murphy Date: Tue, 13 Jan 2026 13:42:16 +1100 Subject: [PATCH 5/9] fix: Remove static creds configuration --- internal/cache/s3.go | 87 ++++++++++++--------------------------- internal/cache/s3_test.go | 16 +++---- 2 files changed, 36 insertions(+), 67 deletions(-) diff --git a/internal/cache/s3.go b/internal/cache/s3.go index 90be3b59..5b6760f6 100644 --- a/internal/cache/s3.go +++ b/internal/cache/s3.go @@ -25,14 +25,12 @@ func init() { } type S3Config struct { - Endpoint string `hcl:"endpoint" help:"S3 endpoint URL (e.g., s3.amazonaws.com or localhost:9000)."` - AccessKeyID string `hcl:"access-key-id,optional" help:"S3 access key ID (optional, uses AWS credential chain if not provided)."` - SecretAccessKey string `hcl:"secret-access-key,optional" help:"S3 secret access key (optional, uses AWS credential chain if not provided)."` - Bucket string `hcl:"bucket" help:"S3 bucket name."` - Region string `hcl:"region,optional" help:"S3 region (defaults to us-west-2)."` - UseSSL *bool `hcl:"use-ssl,optional" help:"Use SSL for S3 connections (defaults to true)."` - SkipSSLVerify bool `hcl:"skip-ssl-verify,optional" help:"Skip SSL certificate verification (defaults to false)."` - MaxTTL time.Duration `hcl:"max-ttl,optional" help:"Maximum time-to-live for entries in the S3 cache (defaults to 1 hour)."` + Endpoint string `hcl:"endpoint" help:"S3 endpoint URL (e.g., s3.amazonaws.com or localhost:9000)."` + Bucket string `hcl:"bucket" help:"S3 bucket name."` + Region string `hcl:"region,optional" help:"S3 region (defaults to us-west-2)."` + UseSSL *bool `hcl:"use-ssl,optional" help:"Use SSL for S3 connections (defaults to true)."` + SkipSSLVerify bool `hcl:"skip-ssl-verify,optional" help:"Skip SSL certificate verification (defaults to false)."` + MaxTTL time.Duration `hcl:"max-ttl,optional" help:"Maximum time-to-live for entries in the S3 cache (defaults to 1 hour)."` } type S3 struct { @@ -43,20 +41,11 @@ type S3 struct { var _ Cache = (*S3)(nil) -// credentialMode returns a string indicating which credential mode is being used. -func credentialMode(config S3Config) string { - if config.AccessKeyID != "" && config.SecretAccessKey != "" { - return "static" - } - return "aws-chain" -} - // NewS3 creates a new S3-based cache instance using the minio SDK. // // config.Endpoint and config.Bucket MUST be set. // -// If config.AccessKeyID and config.SecretAccessKey are provided, static credentials will be used. -// Otherwise, the standard AWS credential chain will be used, which includes: +// The standard AWS credential chain is used for authentication, which includes: // 1. Environment variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN) // 2. AWS credentials file (~/.aws/credentials) // 3. IAM role from EC2 instance metadata or ECS container credentials @@ -91,19 +80,17 @@ func NewS3(ctx context.Context, config S3Config) (*S3, error) { "bucket", config.Bucket, "region", config.Region, "use-ssl", useSSL, - "max-ttl", config.MaxTTL, - "credentials-mode", credentialMode(config)) + "max-ttl", config.MaxTTL) - // Determine credential provider and optional custom transport - var creds *credentials.Credentials - var transport http.RoundTripper + // Create default transport for credential chain + defaultTransport, err := minio.DefaultTransport(useSSL) + if err != nil { + return nil, errors.Errorf("failed to create default transport: %w", err) + } - // Only create custom transport if we need to skip SSL verification + // Apply SSL verification settings if needed + var transport http.RoundTripper if config.SkipSSLVerify { - defaultTransport, err := minio.DefaultTransport(useSSL) - if err != nil { - return nil, errors.Errorf("failed to create default transport: %w", err) - } // Clone the default transport and disable SSL verification customTransport := defaultTransport.Clone() if customTransport.TLSClientConfig == nil { @@ -115,40 +102,20 @@ func NewS3(ctx context.Context, config S3Config) (*S3, error) { } customTransport.TLSClientConfig.InsecureSkipVerify = true transport = customTransport + defaultTransport = customTransport } - switch { - case config.AccessKeyID != "" && config.SecretAccessKey != "": - // Use static credentials if both are provided - creds = credentials.NewStaticV4(config.AccessKeyID, config.SecretAccessKey, "") - case config.AccessKeyID != "" || config.SecretAccessKey != "": - // Error if only one is provided - return nil, errors.New("both access-key-id and secret-access-key must be provided together, or neither for credential chain") - default: - // Use AWS credential chain if neither is provided - defaultTransport, err := minio.DefaultTransport(useSSL) - if err != nil { - return nil, errors.Errorf("failed to create default transport: %w", err) - } - if transport != nil { - // Use custom transport if already set (for SkipSSLVerify) - var ok bool - defaultTransport, ok = transport.(*http.Transport) - if !ok { - return nil, errors.New("transport is not an *http.Transport") - } - } - creds = credentials.NewChainCredentials( - []credentials.Provider{ - &credentials.EnvAWS{}, // Check AWS environment variables - &credentials.FileAWSCredentials{}, // Check ~/.aws/credentials - &credentials.IAM{ - Client: &http.Client{ - Transport: defaultTransport, - }, - }, // Check EC2 instance metadata or ECS container credentials - }) - } + // Use AWS credential chain + creds := credentials.NewChainCredentials( + []credentials.Provider{ + &credentials.EnvAWS{}, // Check AWS environment variables + &credentials.FileAWSCredentials{}, // Check ~/.aws/credentials + &credentials.IAM{ + Client: &http.Client{ + Transport: defaultTransport, + }, + }, // Check EC2 instance metadata or ECS container credentials + }) // Create minio client options options := &minio.Options{ diff --git a/internal/cache/s3_test.go b/internal/cache/s3_test.go index 266e7e14..a1d3c23f 100644 --- a/internal/cache/s3_test.go +++ b/internal/cache/s3_test.go @@ -160,15 +160,17 @@ func TestS3Cache(t *testing.T) { } } + // Set credentials via environment variables for the AWS credential chain + t.Setenv("AWS_ACCESS_KEY_ID", minioUsername) + t.Setenv("AWS_SECRET_ACCESS_KEY", minioPassword) + useSSL := false c, err := cache.NewS3(ctx, cache.S3Config{ - Endpoint: minioEndpoint, - AccessKeyID: minioUsername, - SecretAccessKey: minioPassword, - Bucket: minioBucket, - Region: "", - UseSSL: &useSSL, // MinIO container serves HTTP, not HTTPS - MaxTTL: 100 * time.Millisecond, + Endpoint: minioEndpoint, + Bucket: minioBucket, + Region: "", + UseSSL: &useSSL, // MinIO container serves HTTP, not HTTPS + MaxTTL: 100 * time.Millisecond, }) assert.NoError(t, err) return c From 468753dd5f505c54d57fd37c497ffb783957d14d Mon Sep 17 00:00:00 2001 From: John Murphy Date: Tue, 13 Jan 2026 13:49:43 +1100 Subject: [PATCH 6/9] fix: Input defaults --- internal/cache/s3.go | 37 ++++++++----------------------------- internal/cache/s3_test.go | 2 +- 2 files changed, 9 insertions(+), 30 deletions(-) diff --git a/internal/cache/s3.go b/internal/cache/s3.go index 5b6760f6..2bd55254 100644 --- a/internal/cache/s3.go +++ b/internal/cache/s3.go @@ -25,12 +25,12 @@ func init() { } type S3Config struct { - Endpoint string `hcl:"endpoint" help:"S3 endpoint URL (e.g., s3.amazonaws.com or localhost:9000)."` Bucket string `hcl:"bucket" help:"S3 bucket name."` - Region string `hcl:"region,optional" help:"S3 region (defaults to us-west-2)."` - UseSSL *bool `hcl:"use-ssl,optional" help:"Use SSL for S3 connections (defaults to true)."` - SkipSSLVerify bool `hcl:"skip-ssl-verify,optional" help:"Skip SSL certificate verification (defaults to false)."` - MaxTTL time.Duration `hcl:"max-ttl,optional" help:"Maximum time-to-live for entries in the S3 cache (defaults to 1 hour)."` + Endpoint string `hcl:"endpoint,optional" help:"S3 endpoint URL (e.g., s3.amazonaws.com or localhost:9000)." default:"s3.amazonaws.com"` + Region string `hcl:"region,optional" help:"S3 region (defaults to us-west-2)." default:"us-west-2"` + UseSSL bool `hcl:"use-ssl,optional" help:"Use SSL for S3 connections (defaults to true)." default:"true"` + SkipSSLVerify bool `hcl:"skip-ssl-verify,optional" help:"Skip SSL certificate verification (defaults to false)." default:"false"` + MaxTTL time.Duration `hcl:"max-ttl,optional" help:"Maximum time-to-live for entries in the S3 cache (defaults to 1 hour)." default:"1h"` } type S3 struct { @@ -54,36 +54,15 @@ var _ Cache = (*S3)(nil) // Metadata (headers and expiration time) are stored as object user metadata. The implementation // uses the lightweight minio-go SDK to reduce overhead compared to the AWS SDK. func NewS3(ctx context.Context, config S3Config) (*S3, error) { - // Validate config - if config.Endpoint == "" { - return nil, errors.New("endpoint is required") - } - if config.Bucket == "" { - return nil, errors.New("bucket is required") - } - - // Apply defaults only for zero values - if config.Region == "" { - config.Region = "us-west-2" - } - if config.MaxTTL == 0 { - config.MaxTTL = time.Hour - } - // UseSSL defaults to true if not explicitly set - useSSL := true - if config.UseSSL != nil { - useSSL = *config.UseSSL - } - logging.FromContext(ctx).InfoContext(ctx, "Constructing S3 cache", "endpoint", config.Endpoint, "bucket", config.Bucket, "region", config.Region, - "use-ssl", useSSL, + "use-ssl", config.UseSSL, "max-ttl", config.MaxTTL) // Create default transport for credential chain - defaultTransport, err := minio.DefaultTransport(useSSL) + defaultTransport, err := minio.DefaultTransport(config.UseSSL) if err != nil { return nil, errors.Errorf("failed to create default transport: %w", err) } @@ -120,7 +99,7 @@ func NewS3(ctx context.Context, config S3Config) (*S3, error) { // Create minio client options options := &minio.Options{ Creds: creds, - Secure: useSSL, + Secure: config.UseSSL, Region: config.Region, } diff --git a/internal/cache/s3_test.go b/internal/cache/s3_test.go index a1d3c23f..53e4750a 100644 --- a/internal/cache/s3_test.go +++ b/internal/cache/s3_test.go @@ -169,7 +169,7 @@ func TestS3Cache(t *testing.T) { Endpoint: minioEndpoint, Bucket: minioBucket, Region: "", - UseSSL: &useSSL, // MinIO container serves HTTP, not HTTPS + UseSSL: useSSL, // MinIO container serves HTTP, not HTTPS MaxTTL: 100 * time.Millisecond, }) assert.NoError(t, err) From 9cdd657de5de7cfb05d72326d67084d9a6f90987 Mon Sep 17 00:00:00 2001 From: John Murphy Date: Tue, 13 Jan 2026 14:10:13 +1100 Subject: [PATCH 7/9] feat: Remove the buffer and add a pipe for streaming to s3 --- internal/cache/s3.go | 55 +++++++++++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/internal/cache/s3.go b/internal/cache/s3.go index 2bd55254..a3b5dff0 100644 --- a/internal/cache/s3.go +++ b/internal/cache/s3.go @@ -1,7 +1,6 @@ package cache import ( - "bytes" "context" "crypto/tls" "encoding/json" @@ -188,14 +187,22 @@ func (s *S3) Create(ctx context.Context, key Key, headers textproto.MIMEHeader, expiresAt := time.Now().Add(ttl) - return &s3Writer{ + pr, pw := io.Pipe() + + writer := &s3Writer{ s3: s, key: key, - buf: &bytes.Buffer{}, + pipe: pw, expiresAt: expiresAt, headers: headers, ctx: ctx, - }, nil + errCh: make(chan error, 1), + } + + // Start upload in background goroutine + go writer.upload(pr) + + return writer, nil } func (s *S3) Delete(ctx context.Context, key Key) error { @@ -222,22 +229,35 @@ func (s *S3) Delete(ctx context.Context, key Key) error { type s3Writer struct { s3 *S3 key Key - buf *bytes.Buffer + pipe *io.PipeWriter expiresAt time.Time headers textproto.MIMEHeader ctx context.Context + errCh chan error } func (w *s3Writer) Write(p []byte) (int, error) { - return errors.WithStack2(w.buf.Write(p)) + return errors.WithStack2(w.pipe.Write(p)) } func (w *s3Writer) Close() error { - // Check if context was cancelled - if err := w.ctx.Err(); err != nil { - return errors.Wrap(err, "create operation cancelled") + // Close the pipe writer to signal EOF to the reader + if err := w.pipe.Close(); err != nil { + return errors.Wrap(err, "failed to close pipe") } + // Wait for upload to complete and get any error + err := <-w.errCh + if err != nil { + return err + } + + return nil +} + +func (w *s3Writer) upload(pr *io.PipeReader) { + defer pr.Close() + objectName := w.key.String() // Prepare user metadata @@ -246,7 +266,8 @@ func (w *s3Writer) Close() error { // Store expiration time expiresAtBytes, err := w.expiresAt.MarshalText() if err != nil { - return errors.Errorf("failed to marshal expiration time: %w", err) + w.errCh <- errors.Errorf("failed to marshal expiration time: %w", err) + return } userMetadata["Expires-At"] = string(expiresAtBytes) @@ -254,25 +275,27 @@ func (w *s3Writer) Close() error { if len(w.headers) > 0 { headersJSON, err := json.Marshal(w.headers) if err != nil { - return errors.Errorf("failed to marshal headers: %w", err) + w.errCh <- errors.Errorf("failed to marshal headers: %w", err) + return } userMetadata["Headers"] = string(headersJSON) } - // Upload object + // Upload object with streaming (size -1 means unknown size, will use chunked encoding) _, err = w.s3.client.PutObject( w.ctx, w.s3.config.Bucket, objectName, - w.buf, - int64(w.buf.Len()), + pr, + -1, minio.PutObjectOptions{ UserMetadata: userMetadata, }, ) if err != nil { - return errors.Errorf("failed to put object: %w", err) + w.errCh <- errors.Errorf("failed to put object: %w", err) + return } - return nil + w.errCh <- nil } From a42440a4daa054a70e741552f10e3671a9435894 Mon Sep 17 00:00:00 2001 From: John Murphy Date: Tue, 13 Jan 2026 14:24:18 +1100 Subject: [PATCH 8/9] fix: Make the disk and s3 cache pathing more consistent --- internal/cache/s3.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/internal/cache/s3.go b/internal/cache/s3.go index a3b5dff0..d79f42f6 100644 --- a/internal/cache/s3.go +++ b/internal/cache/s3.go @@ -136,8 +136,14 @@ func (s *S3) Close() error { return nil } +func (s *S3) keyToPath(key Key) string { + hexKey := key.String() + // Use first two hex digits as directory, full hex as filename + return hexKey[:2] + "/" + hexKey +} + func (s *S3) Open(ctx context.Context, key Key) (io.ReadCloser, textproto.MIMEHeader, error) { - objectName := key.String() + objectName := s.keyToPath(key) // Get object info to check metadata objInfo, err := s.client.StatObject(ctx, s.config.Bucket, objectName, minio.StatObjectOptions{}) @@ -206,7 +212,7 @@ func (s *S3) Create(ctx context.Context, key Key, headers textproto.MIMEHeader, } func (s *S3) Delete(ctx context.Context, key Key) error { - objectName := key.String() + objectName := s.keyToPath(key) // Check if object exists first _, err := s.client.StatObject(ctx, s.config.Bucket, objectName, minio.StatObjectOptions{}) @@ -258,7 +264,7 @@ func (w *s3Writer) Close() error { func (w *s3Writer) upload(pr *io.PipeReader) { defer pr.Close() - objectName := w.key.String() + objectName := w.s3.keyToPath(w.key) // Prepare user metadata userMetadata := make(map[string]string) From c2cd5c80cc0fcf5f1dcbcbff51222968cefb5ad3 Mon Sep 17 00:00:00 2001 From: John Murphy Date: Tue, 13 Jan 2026 14:38:29 +1100 Subject: [PATCH 9/9] fix: Remove unnecessary ObjectStat operation --- internal/cache/s3.go | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/internal/cache/s3.go b/internal/cache/s3.go index d79f42f6..8960719b 100644 --- a/internal/cache/s3.go +++ b/internal/cache/s3.go @@ -214,17 +214,7 @@ func (s *S3) Create(ctx context.Context, key Key, headers textproto.MIMEHeader, func (s *S3) Delete(ctx context.Context, key Key) error { objectName := s.keyToPath(key) - // Check if object exists first - _, err := s.client.StatObject(ctx, s.config.Bucket, objectName, minio.StatObjectOptions{}) - if err != nil { - errResponse := minio.ToErrorResponse(err) - if errResponse.Code == "NoSuchKey" { - return os.ErrNotExist - } - return errors.Errorf("failed to stat object: %w", err) - } - - err = s.client.RemoveObject(ctx, s.config.Bucket, objectName, minio.RemoveObjectOptions{}) + err := s.client.RemoveObject(ctx, s.config.Bucket, objectName, minio.RemoveObjectOptions{}) if err != nil { return errors.Errorf("failed to remove object: %w", err) }