From 8c290e7e401196839f8284f0388f10dfa855d6f1 Mon Sep 17 00:00:00 2001 From: Xe Iaso Date: Sat, 25 May 2024 07:29:40 -0400 Subject: [PATCH] tigris: helper functions for dealing with Tigris Signed-off-by: Xe Iaso --- cmd/uploud/main.go | 109 ++++++++++++++++++++++++++++++----- cmd/xedn/uplodr/main.go | 17 +----- internal/tigris.go | 21 ------- tigris/tigris.go | 125 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 222 insertions(+), 50 deletions(-) delete mode 100644 internal/tigris.go create mode 100644 tigris/tigris.go diff --git a/cmd/uploud/main.go b/cmd/uploud/main.go index 91752c75..cef798d4 100644 --- a/cmd/uploud/main.go +++ b/cmd/uploud/main.go @@ -3,6 +3,9 @@ package main import ( "context" + "crypto/md5" + "crypto/sha256" + "encoding/base64" "flag" "fmt" "image" @@ -20,6 +23,7 @@ import ( "github.com/disintegration/imaging" "within.website/x/internal" "within.website/x/internal/avif" + "within.website/x/tigris" ) var ( @@ -226,7 +230,7 @@ func main() { log.Fatal(err) } - s3c, err := internal.TigrisClient(context.Background()) + s3c, err := tigris.Client(context.Background()) if err != nil { log.Fatal(err) } @@ -239,12 +243,33 @@ func main() { } defer fin.Close() - _, err = s3c.PutObject(ctx, &s3.PutObjectInput{ - Body: fin, - Bucket: b2Bucket, - Key: aws.String(flag.Arg(1) + "/" + finfo.Name()), - ContentType: aws.String(mimeTypes[filepath.Ext(finfo.Name())]), - }) + st, err := fin.Stat() + if err != nil { + log.Fatal(err) + } + + shaSum, err := hashFileSha256(fin) + if err != nil { + log.Fatal(err) + } + + md5Sum, err := hashFileMD5(fin) + if err != nil { + log.Fatal(err) + } + + _, err = s3c.PutObject(ctx, + &s3.PutObjectInput{ + Body: fin, + Bucket: b2Bucket, + Key: aws.String(flag.Arg(1) + "/" + finfo.Name()), + ContentType: aws.String(mimeTypes[filepath.Ext(finfo.Name())]), + ContentLength: aws.Int64(st.Size()), + ChecksumSHA256: aws.String(shaSum), + ContentMD5: aws.String(md5Sum), + }, + tigris.WithCreateObjectIfNotExists(), + ) if err != nil { log.Fatal(err) } @@ -252,11 +277,67 @@ func main() { } var mimeTypes = map[string]string{ - ".avif": "image/avif", - ".webp": "image/webp", - ".jpg": "image/jpeg", - ".png": "image/png", - ".svg": "image/svg+xml", - ".wasm": "application/wasm", - ".css": "text/css", + ".avif": "image/avif", + ".webp": "image/webp", + ".jpg": "image/jpeg", + ".png": "image/png", + ".svg": "image/svg+xml", + ".wasm": "application/wasm", + ".css": "text/css", + ".ts": "video/mp2t", + ".js": "application/javascript", + ".html": "text/html", + ".json": "application/json", + ".txt": "text/plain", + ".md": "text/markdown", + ".xml": "application/xml", + ".zip": "application/zip", + ".gz": "application/gzip", + ".tar": "application/x-tar", + ".pdf": "application/pdf", + ".mp4": "video/mp4", + ".webm": "video/webm", + ".ogg": "audio/ogg", + ".mp3": "audio/mpeg", + ".wav": "audio/wav", + ".flac": "audio/flac", + ".aac": "audio/aac", + ".m4a": "audio/mp4", + ".opus": "audio/opus", + ".ico": "image/x-icon", + ".otf": "font/otf", + ".ttf": "font/ttf", + ".woff": "font/woff", + ".woff2": "font/woff2", + ".eot": "application/vnd.ms-fontobject", +} + +// hashFileSha256 hashes a file with Sha256 and returns the hash as a base64 encoded string. +func hashFileSha256(fin *os.File) (string, error) { + h := sha256.New() + if _, err := io.Copy(h, fin); err != nil { + return "", err + } + + // rewind the file + if _, err := fin.Seek(0, io.SeekStart); err != nil { + return "", err + } + + return base64.StdEncoding.EncodeToString(h.Sum(nil)), nil +} + +// hashFileMD5 hashes a file with MD5 and returns the hash as a base64 encoded string. +func hashFileMD5(fin *os.File) (string, error) { + h := md5.New() + if _, err := io.Copy(h, fin); err != nil { + return "", err + } + + // rewind the file + if _, err := fin.Seek(0, io.SeekStart); err != nil { + return "", err + } + + return base64.StdEncoding.EncodeToString(h.Sum(nil)), nil } diff --git a/cmd/xedn/uplodr/main.go b/cmd/xedn/uplodr/main.go index e226ddc9..e1a9c101 100644 --- a/cmd/xedn/uplodr/main.go +++ b/cmd/xedn/uplodr/main.go @@ -18,7 +18,6 @@ import ( "time" "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/chai2010/webp" @@ -27,6 +26,7 @@ import ( "within.website/x/cmd/xedn/uplodr/pb" "within.website/x/internal" "within.website/x/internal/avif" + "within.website/x/tigris" ) var ( @@ -89,7 +89,7 @@ type Server struct { } func New(ctx context.Context) (*Server, error) { - tc, err := mkTigrisClient(ctx) + tc, err := tigris.Client(ctx) if err != nil { return nil, fmt.Errorf("failed to create Tigris client: %w", err) } @@ -303,19 +303,6 @@ var mimeTypes = map[string]string{ ".css": "text/css", } -func mkTigrisClient(ctx context.Context) (*s3.Client, error) { - cfg, err := config.LoadDefaultConfig(ctx) - if err != nil { - return nil, fmt.Errorf("failed to load Tigris config: %w", err) - } - cfg.Region = "auto" - - return s3.NewFromConfig(cfg, func(o *s3.Options) { - o.BaseEndpoint = aws.String("https://fly.storage.tigris.dev") - o.Region = "auto" - }), nil -} - func mkB2Client() *s3.Client { s3Config := aws.Config{ Credentials: credentials.NewStaticCredentialsProvider(*b2KeyID, *b2KeySecret, ""), diff --git a/internal/tigris.go b/internal/tigris.go deleted file mode 100644 index 91a90e36..00000000 --- a/internal/tigris.go +++ /dev/null @@ -1,21 +0,0 @@ -package internal - -import ( - "context" - "fmt" - - "github.com/aws/aws-sdk-go-v2/aws" - awsConfig "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/s3" -) - -func TigrisClient(ctx context.Context) (*s3.Client, error) { - cfg, err := awsConfig.LoadDefaultConfig(ctx) - if err != nil { - return nil, fmt.Errorf("failed to load Tigris config: %w", err) - } - - return s3.NewFromConfig(cfg, func(o *s3.Options) { - o.BaseEndpoint = aws.String("https://fly.storage.tigris.dev") - }), nil -} diff --git a/tigris/tigris.go b/tigris/tigris.go new file mode 100644 index 00000000..009fa64d --- /dev/null +++ b/tigris/tigris.go @@ -0,0 +1,125 @@ +// Package tigris contains a Tigris client and helpers for interacting with Tigris. +// +// Tigris is a cloud storage service that provides a simple, scalable, and secure object storage solution. It is based on the S3 API, but has additional features that need these helpers. +package tigris + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + awsConfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go/transport/http" +) + +// WithHeader sets an arbitrary HTTP header on the request. +func WithHeader(key, value string) func(*s3.Options) { + return func(options *s3.Options) { + options.APIOptions = append(options.APIOptions, http.AddHeaderValue(key, value)) + } +} + +// Region is a Tigris region from the documentation. +// +// https://www.tigrisdata.com/docs/concepts/regions/ +type Region string + +// Possible Tigris regions. +const ( + FRA Region = "fra" // Frankfurt, Germany + GRU Region = "gru" // São Paulo, Brazil + HKG Region = "hkg" // Hong Kong, China + IAD Region = "iad" // Ashburn, Virginia, USA + JNB Region = "jnb" // Johannesburg, South Africa + LHR Region = "lhr" // London, UK + MAD Region = "mad" // Madrid, Spain + NRT Region = "nrt" // Tokyo (Narita), Japan + ORD Region = "ord" // Chicago, Illinois, USA + SIN Region = "sin" // Singapore + SJC Region = "sjc" // San Jose, California, USA + SYD Region = "syd" // Sydney, Australia +) + +// WithStaticReplicationRegions sets the regions where the object will be replicated. +// +// Note that this will cause you to be charged multiple times for the same object, once per region. +func WithStaticReplicationRegions(regions []Region) func(*s3.Options) { + regionsString := make([]string, 0, len(regions)) + for _, r := range regions { + regionsString = append(regionsString, string(r)) + } + + return WithHeader("X-Tigris-Regions", strings.Join(regionsString, ",")) +} + +// WithQuery lets you filter objects in a ListObjectsV2 request. +// +// This functions like the WHERE clause in SQL, but for S3 objects. For more information, see the Tigris documentation[1]. +// +// [1]: https://www.tigrisdata.com/docs/objects/query-metadata/ +func WithQuery(query string) func(*s3.Options) { + return WithHeader("X-Tigris-Query", query) +} + +// WithCreateIfNotExists will create the object if it doesn't exist. +// +// See the Tigris documentation[1] for more information. +// +// [1]: https://www.tigrisdata.com/docs/objects/conditionals/ +func WithCreateObjectIfNotExists() func(*s3.Options) { + return WithHeader("If-Match", `""`) +} + +// WithIfEtagMatches sets the ETag that the object must match. +// +// See the Tigris documentation[1] for more information. +// +// [1]: https://www.tigrisdata.com/docs/objects/conditionals/ +func WithIfEtagMatches(etag string) func(*s3.Options) { + return WithHeader("If-Match", etag) +} + +// WithModifiedSince lets you proceed with operation if object was modified after provided date (RFC1123). +// +// See the Tigris documentation[1] for more information. +// +// [1]: https://www.tigrisdata.com/docs/objects/conditionals/ +func WithModifiedSince(modifiedSince time.Time) func(*s3.Options) { + return WithHeader("If-Modified-Since", modifiedSince.Format(time.RFC1123)) +} + +// WithUnmodifiedSince lets you proceed with operation if object was not modified after provided date (RFC1123). +// +// See the Tigris documentation[1] for more information. +// +// [1]: https://www.tigrisdata.com/docs/objects/conditionals/ +func WithUnmodifiedSince(unmodifiedSince time.Time) func(*s3.Options) { + return WithHeader("If-Unmodified-Since", unmodifiedSince.Format(time.RFC1123)) +} + +// WithCompareAndSwap tells Tigris to skip the cache and read the object from its designated region. +// +// This is only used on GET requests. +// +// See the Tigris documentation[1] for more information. +// +// [1]: https://www.tigrisdata.com/docs/objects/conditionals/ +func WithCompareAndSwap() func(*s3.Options) { + return WithHeader("X-Tigris-CAS", "true") +} + +// Client returns a new S3 client wired up for Tigris. +func Client(ctx context.Context) (*s3.Client, error) { + cfg, err := awsConfig.LoadDefaultConfig(ctx) + if err != nil { + return nil, fmt.Errorf("failed to load Tigris config: %w", err) + } + + return s3.NewFromConfig(cfg, func(o *s3.Options) { + o.BaseEndpoint = aws.String("https://fly.storage.tigris.dev") + o.Region = "auto" + }), nil +}