Merge #570

570: Feat support content encoding  r=curquiza a=Ja7ad

# Pull Request

## Related issue
Fixes #566 

## What does this PR do?

This PR support content encoding `gzip`, `deflate`, `brotli`.


```go
client := meilisearch.New("127.0.0.1:7700", meilisearch.WithContentEncoding(meilisearch.GzipEncoding, meilisearch.DefaultCompression))
```

Note: This PR is not breaking changes.

## PR checklist
Please check if your PR fulfills the following requirements:
- [x] Does this PR fix an existing issue, or have you listed the changes applied in the PR description (and why they are needed)?
- [x] Have you read the contributing guidelines?
- [x] Have you made sure that the title is accurate and descriptive of the changes?

Thank you so much for contributing to Meilisearch!


Co-authored-by: Javad <[email protected]>
Co-authored-by: Javad Rajabzadeh <[email protected]>

Author: meili-bors[bot]

client.go

@@ -13,17 +13,18 @@ import (
 )
 
 type client struct {
-	client     *http.Client
-	host       string
-	apiKey     string
-	bufferPool *sync.Pool
+	client          *http.Client
+	host            string
+	apiKey          string
+	bufferPool      *sync.Pool
+	encoder         encoder
+	contentEncoding ContentEncoding
 }
 
 type internalRequest struct {
-	endpoint    string
-	method      string
-	contentType string
-
+	endpoint        string
+	method          string
+	contentType     string
 	withRequest     interface{}
 	withResponse    interface{}
 	withQueryParams map[string]string
@@ -33,8 +34,8 @@ type internalRequest struct {
 	functionName string
 }
 
-func newClient(cli *http.Client, host, apiKey string) *client {
-	return &client{
+func newClient(cli *http.Client, host, apiKey string, ce ContentEncoding, cl EncodingCompressionLevel) *client {
+	c := &client{
 		client: cli,
 		host:   host,
 		apiKey: apiKey,
@@ -44,6 +45,13 @@ func newClient(cli *http.Client, host, apiKey string) *client {
 			},
 		},
 	}
+
+	if !ce.IsZero() {
+		c.contentEncoding = ce
+		c.encoder = newEncoding(ce, cl)
+	}
+
+	return c
 }
 
 func (c *client) executeRequest(ctx context.Context, req *internalRequest) error {
@@ -57,6 +65,7 @@ func (c *client) executeRequest(ctx context.Context, req *internalRequest) error
 			Message: "empty meilisearch message",
 		},
 		StatusCodeExpected: req.acceptedStatusCodes,
+		encoder:            c.encoder,
 	}
 
 	resp, err := c.sendRequest(ctx, req, internalError)
@@ -118,10 +127,11 @@ func (c *client) sendRequest(
 		}
 
 		rawRequest := req.withRequest
+
+		buf := c.bufferPool.Get().(*bytes.Buffer)
+		buf.Reset()
+
 		if b, ok := rawRequest.([]byte); ok {
-			// If the request body is already a []byte then use it directly
-			buf := c.bufferPool.Get().(*bytes.Buffer)
-			buf.Reset()
 			buf.Write(b)
 			body = buf
 		} else if reader, ok := rawRequest.(io.Reader); ok {
@@ -136,22 +146,31 @@ func (c *client) sendRequest(
 			if marshaler, ok := rawRequest.(json.Marshaler); ok {
 				data, err = marshaler.MarshalJSON()
 				if err != nil {
-					return nil, internalError.WithErrCode(ErrCodeMarshalRequest, fmt.Errorf("failed to marshal with MarshalJSON: %w", err))
+					return nil, internalError.WithErrCode(ErrCodeMarshalRequest,
+						fmt.Errorf("failed to marshal with MarshalJSON: %w", err))
 				}
 				if data == nil {
-					return nil, internalError.WithErrCode(ErrCodeMarshalRequest, errors.New("MarshalJSON returned nil data"))
+					return nil, internalError.WithErrCode(ErrCodeMarshalRequest,
+						errors.New("MarshalJSON returned nil data"))
 				}
 			} else {
 				data, err = json.Marshal(rawRequest)
 				if err != nil {
-					return nil, internalError.WithErrCode(ErrCodeMarshalRequest, fmt.Errorf("failed to marshal with json.Marshal: %w", err))
+					return nil, internalError.WithErrCode(ErrCodeMarshalRequest,
+						fmt.Errorf("failed to marshal with json.Marshal: %w", err))
 				}
 			}
-			buf := c.bufferPool.Get().(*bytes.Buffer)
-			buf.Reset()
 			buf.Write(data)
 			body = buf
 		}
+
+		if !c.contentEncoding.IsZero() {
+			body, err = c.encoder.Encode(body)
+			if err != nil {
+				return nil, internalError.WithErrCode(ErrCodeMarshalRequest,
+					fmt.Errorf("failed to marshal with json.Marshal: %w", err))
+			}
+		}
 	}
 
 	// Create the HTTP request
@@ -168,6 +187,14 @@ func (c *client) sendRequest(
 		request.Header.Set("Authorization", "Bearer "+c.apiKey)
 	}
 
+	if req.withResponse != nil && !c.contentEncoding.IsZero() {
+		request.Header.Set("Accept-Encoding", c.contentEncoding.String())
+	}
+
+	if req.withRequest != nil && !c.contentEncoding.IsZero() {
+		request.Header.Set("Content-Encoding", c.contentEncoding.String())
+	}
+
 	request.Header.Set("User-Agent", GetQualifiedVersion())
 
 	resp, err := c.client.Do(request)
@@ -210,18 +237,23 @@ func (c *client) handleStatusCode(req *internalRequest, statusCode int, body []b
 
 func (c *client) handleResponse(req *internalRequest, body []byte, internalError *Error) (err error) {
 	if req.withResponse != nil {
-
-		internalError.ResponseToString = string(body)
-
-		var err error
-		if resp, ok := req.withResponse.(json.Unmarshaler); ok {
-			err = resp.UnmarshalJSON(body)
-			req.withResponse = resp
+		if !c.contentEncoding.IsZero() {
+			if err := c.encoder.Decode(body, req.withResponse); err != nil {
+				return internalError.WithErrCode(ErrCodeResponseUnmarshalBody, err)
+			}
 		} else {
-			err = json.Unmarshal(body, req.withResponse)
-		}
-		if err != nil {
-			return internalError.WithErrCode(ErrCodeResponseUnmarshalBody, err)
+			internalError.ResponseToString = string(body)
+
+			var err error
+			if resp, ok := req.withResponse.(json.Unmarshaler); ok {
+				err = resp.UnmarshalJSON(body)
+				req.withResponse = resp
+			} else {
+				err = json.Unmarshal(body, req.withResponse)
+			}
+			if err != nil {
+				return internalError.WithErrCode(ErrCodeResponseUnmarshalBody, err)
+			}
 		}
 	}
 	return nil

client_test.go

@@ -1,10 +1,12 @@
 package meilisearch
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
 	"github.com/stretchr/testify/require"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"strings"
@@ -29,9 +31,65 @@ func TestExecuteRequest(t *testing.T) {
 		if r.Method == http.MethodGet && r.URL.Path == "/test-get" {
 			w.WriteHeader(http.StatusOK)
 			_, _ = w.Write([]byte(`{"message":"get successful"}`))
+		} else if r.Method == http.MethodGet && r.URL.Path == "/test-get-encoding" {
+			encode := r.Header.Get("Accept-Encoding")
+			if len(encode) != 0 {
+				enc := newEncoding(ContentEncoding(encode), DefaultCompression)
+				d := &mockData{Name: "foo", Age: 30}
+
+				b, err := json.Marshal(d)
+				require.NoError(t, err)
+
+				res, err := enc.Encode(bytes.NewReader(b))
+				require.NoError(t, err)
+				_, _ = w.Write(res.Bytes())
+				w.WriteHeader(http.StatusOK)
+				return
+			}
+			_, _ = w.Write([]byte("invalid message"))
+			w.WriteHeader(http.StatusInternalServerError)
+		} else if r.Method == http.MethodPost && r.URL.Path == "/test-req-resp-encoding" {
+			accept := r.Header.Get("Accept-Encoding")
+			ce := r.Header.Get("Content-Encoding")
+
+			reqEnc := newEncoding(ContentEncoding(ce), DefaultCompression)
+			respEnc := newEncoding(ContentEncoding(accept), DefaultCompression)
+			req := new(mockData)
+
+			if len(ce) != 0 {
+				b, err := io.ReadAll(r.Body)
+				require.NoError(t, err)
+
+				err = reqEnc.Decode(b, req)
+				require.NoError(t, err)
+			}
+
+			if len(accept) != 0 {
+				d, err := json.Marshal(req)
+				require.NoError(t, err)
+				res, err := respEnc.Encode(bytes.NewReader(d))
+				require.NoError(t, err)
+				_, _ = w.Write(res.Bytes())
+				w.WriteHeader(http.StatusOK)
+			}
 		} else if r.Method == http.MethodPost && r.URL.Path == "/test-post" {
 			w.WriteHeader(http.StatusCreated)
-			_, _ = w.Write([]byte(`{"message":"post successful"}`))
+			msg := []byte(`{"message":"post successful"}`)
+			_, _ = w.Write(msg)
+
+		} else if r.Method == http.MethodPost && r.URL.Path == "/test-post-encoding" {
+			w.WriteHeader(http.StatusCreated)
+			msg := []byte(`{"message":"post successful"}`)
+
+			enc := r.Header.Get("Accept-Encoding")
+			if len(enc) != 0 {
+				e := newEncoding(ContentEncoding(enc), DefaultCompression)
+				b, err := e.Encode(bytes.NewReader(msg))
+				require.NoError(t, err)
+				_, _ = w.Write(b.Bytes())
+				return
+			}
+			_, _ = w.Write(msg)
 		} else if r.URL.Path == "/test-bad-request" {
 			w.WriteHeader(http.StatusBadRequest)
 			_, _ = w.Write([]byte(`{"message":"bad request"}`))
@@ -47,13 +105,12 @@ func TestExecuteRequest(t *testing.T) {
 	}))
 	defer ts.Close()
 
-	client := newClient(&http.Client{}, ts.URL, "testApiKey")
-
 	tests := []struct {
-		name         string
-		internalReq  *internalRequest
-		expectedResp interface{}
-		wantErr      bool
+		name            string
+		internalReq     *internalRequest
+		expectedResp    interface{}
+		contentEncoding ContentEncoding
+		wantErr         bool
 	}{
 		{
 			name: "Successful GET request",
@@ -190,11 +247,108 @@ func TestExecuteRequest(t *testing.T) {
 			expectedResp: nil,
 			wantErr:      true,
 		},
+		{
+			name: "Test request encoding gzip",
+			internalReq: &internalRequest{
+				endpoint:            "/test-post-encoding",
+				method:              http.MethodPost,
+				withRequest:         map[string]string{"key": "value"},
+				contentType:         contentTypeJSON,
+				withResponse:        &mockResponse{},
+				acceptedStatusCodes: []int{http.StatusCreated},
+			},
+			expectedResp:    &mockResponse{Message: "post successful"},
+			contentEncoding: GzipEncoding,
+			wantErr:         false,
+		},
+		{
+			name: "Test request encoding deflate",
+			internalReq: &internalRequest{
+				endpoint:            "/test-post-encoding",
+				method:              http.MethodPost,
+				withRequest:         map[string]string{"key": "value"},
+				contentType:         contentTypeJSON,
+				withResponse:        &mockResponse{},
+				acceptedStatusCodes: []int{http.StatusCreated},
+			},
+			expectedResp:    &mockResponse{Message: "post successful"},
+			contentEncoding: DeflateEncoding,
+			wantErr:         false,
+		},
+		{
+			name: "Test request encoding brotli",
+			internalReq: &internalRequest{
+				endpoint:            "/test-post-encoding",
+				method:              http.MethodPost,
+				withRequest:         map[string]string{"key": "value"},
+				contentType:         contentTypeJSON,
+				withResponse:        &mockResponse{},
+				acceptedStatusCodes: []int{http.StatusCreated},
+			},
+			expectedResp:    &mockResponse{Message: "post successful"},
+			contentEncoding: BrotliEncoding,
+			wantErr:         false,
+		},
+		{
+			name: "Test response decoding gzip",
+			internalReq: &internalRequest{
+				endpoint:            "/test-get-encoding",
+				method:              http.MethodGet,
+				withRequest:         nil,
+				withResponse:        &mockData{},
+				acceptedStatusCodes: []int{http.StatusOK},
+			},
+			expectedResp:    &mockData{Name: "foo", Age: 30},
+			contentEncoding: GzipEncoding,
+			wantErr:         false,
+		},
+		{
+			name: "Test response decoding deflate",
+			internalReq: &internalRequest{
+				endpoint:            "/test-get-encoding",
+				method:              http.MethodGet,
+				withRequest:         nil,
+				withResponse:        &mockData{},
+				acceptedStatusCodes: []int{http.StatusOK},
+			},
+			expectedResp:    &mockData{Name: "foo", Age: 30},
+			contentEncoding: DeflateEncoding,
+			wantErr:         false,
+		},
+		{
+			name: "Test response decoding brotli",
+			internalReq: &internalRequest{
+				endpoint:            "/test-get-encoding",
+				method:              http.MethodGet,
+				withRequest:         nil,
+				withResponse:        &mockData{},
+				acceptedStatusCodes: []int{http.StatusOK},
+			},
+			expectedResp:    &mockData{Name: "foo", Age: 30},
+			contentEncoding: BrotliEncoding,
+			wantErr:         false,
+		},
+		{
+			name: "Test request and response encoding",
+			internalReq: &internalRequest{
+				endpoint:            "/test-req-resp-encoding",
+				method:              http.MethodPost,
+				contentType:         contentTypeJSON,
+				withRequest:         &mockData{Name: "foo", Age: 30},
+				withResponse:        &mockData{},
+				acceptedStatusCodes: []int{http.StatusOK},
+			},
+			expectedResp:    &mockData{Name: "foo", Age: 30},
+			contentEncoding: GzipEncoding,
+			wantErr:         false,
+		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			err := client.executeRequest(context.Background(), tt.internalReq)
+			c := newClient(&http.Client{}, ts.URL, "testApiKey", tt.contentEncoding, DefaultCompression)
+
+			err := c.executeRequest(context.Background(), tt.internalReq)
 			if tt.wantErr {
 				require.Error(t, err)
 			} else {