feat: schemaless HTTP requests (#45)

Add a new sendHttpRequest mutation to send schemaless HTTP requests. It's used as a dynamic GraphQL to REST proxy.

Author: hgiasac

.github/workflows/release.yaml

@@ -51,6 +51,8 @@ jobs:
           tags: ${{ steps.docker-metadata.outputs.tags }}
           labels: ${{ steps.docker-metadata.outputs.labels }}
           platforms: linux/amd64,linux/arm64
+          build-args: |
+            VERSION=${{ steps.get-version.outputs.tagged_version }}
 
   build-cli-and-manifests:
     name: Build the CLI binaries and manifests

Dockerfile

@@ -2,11 +2,16 @@
 FROM golang:1.23 AS builder
 
 WORKDIR /app
+
+ARG VERSION
 COPY ndc-http-schema ./ndc-http-schema
 COPY go.mod go.sum go.work ./
 RUN go mod download
 COPY . .
-RUN CGO_ENABLED=0 go build -v -o ndc-cli ./server
+
+RUN CGO_ENABLED=0 go build \
+  -ldflags "-X github.com/hasura/ndc-http/ndc-http-schema/version.BuildVersion=${VERSION}" \
+  -v -o ndc-cli ./server
 
 # stage 2: production image
 FROM gcr.io/distroless/static-debian12:nonroot

README.md

@@ -12,12 +12,13 @@ The connector can automatically transform OpenAPI 2.0 and 3.0 definitions to NDC
 
 - [No code. Configuration based](#configuration).
 - Composable API collections.
-- [Supported many API specifications](#supported-specs).
+- [Supported many API specifications](./docs/configuration.md#supported-specs).
 - [Supported authentication](./docs/authentication.md).
 - [Supported headers forwarding](./docs/authentication.md#headers-forwarding).
 - [Supported argument presets](./docs/argument_presets.md).
 - [Supported timeout and retry](#timeout-and-retry).
 - Supported concurrency and [sending distributed requests](./docs/distribution.md) to multiple servers.
+- [GraphQL-to-REST proxy](./docs/schemaless_request.md).
 
 **Supported request types**
 
@@ -55,96 +56,19 @@ The connector can automatically transform OpenAPI 2.0 and 3.0 definitions to NDC
 | OAuth 2.0       | ✅        | Built-in support for the `client_credentials` grant. Other grant types require forwarding access tokens from headers by the Hasura engine |
 | mTLS            | ✅        |                                                                                                                                           |
 
-## Quick start
+## Get Started
 
-Start the connector server at http://localhost:8080 using the [JSON Placeholder](https://jsonplaceholder.typicode.com/) APIs.
-
-```go
-go run ./server serve --configuration ./connector/testdata/jsonplaceholder
-```
+Follow the [Quick Start Guide](https://hasura.io/docs/3.0/getting-started/overview/) in Hasura DDN docs. At the `Connect to data` step, choose the `hasura/http` data connector from the dropdown. The connector template includes an example that is ready to run.
 
 ## Documentation
 
-- [NDC HTTP schema](./ndc-http-schema)
+- [Configuration](./docs/configuration.md)
 - [Authentication](./docs/authentication.md)
 - [Argument Presets](./docs/argument_presets.md)
+- [Schemaless Requests](./docs/schemaless_request.md)
 - [Distributed Execution](./docs/distribution.md)
 - [Recipes](https://github.com/hasura/ndc-http-recipes/tree/main): You can find or request pre-built configuration recipes of popular API services here.
-
-## Configuration
-
-The connector reads `config.{json,yaml}` file in the configuration folder. The file contains information about the schema file path and its specification:
-
-```yaml
-files:
-  - file: swagger.json
-    spec: openapi2
-  - file: openapi.yaml
-    spec: openapi3
-    trimPrefix: /v1
-    envPrefix: PET_STORE
-  - file: schema.json
-    spec: ndc
-```
-
-The config of each element follows the [config schema](https://github.com/hasura/ndc-http/ndc-http-schema/blob/main/config.example.yaml) of `ndc-http-schema`.
-
-You can add many API documentation files into the same connector.
-
-> [!IMPORTANT]
-> Conflicted object and scalar types will be ignored. Only the type of the first file is kept in the schema.
-
-### Supported specs
-
-#### OpenAPI
-
-HTTP connector supports both OpenAPI 2 and 3 specifications.
-
-- `oas3`/`openapi3`: OpenAPI 3.0/3.1.
-- `oas2`/`openapi2`: OpenAPI 2.0.
-
-#### HTTP Connector schema
-
-Enum: `ndc`
-
-HTTP schema is the native configuration schema which other specs will be converted to behind the scene. The schema extends the NDC Specification with HTTP configuration and can be converted from other specs by the [NDC HTTP schema CLI](./ndc-http-schema).
-
-### Timeout and retry
-
-The global timeout and retry strategy can be configured in each file:
-
-```yaml
-files:
-  - file: swagger.json
-    spec: oas2
-    timeout:
-      value: 30
-    retry:
-      times:
-        value: 1
-      delay:
-        # delay between each retry in milliseconds
-        value: 500
-      httpStatus: [429, 500, 502, 503]
-```
-
-### JSON Patch
-
-You can add JSON patches to extend API documentation files. HTTP connector supports `merge` and `json6902` strategies. JSON patches can be applied before or after the conversion from OpenAPI to HTTP schema configuration. It will be useful if you need to extend or fix some fields in the API documentation such as server URL.
-
-```yaml
-files:
-  - file: openapi.yaml
-    spec: oas3
-    patchBefore:
-      - path: patch-before.yaml
-        strategy: merge
-    patchAfter:
-      - path: patch-after.yaml
-        strategy: json6902
-```
-
-See [the example](./ndc-http-schema/command/testdata/patch) for more context.
+- [NDC HTTP schema](./ndc-http-schema)
 
 ## License
 

connector/connector.go

@@ -9,18 +9,20 @@ import (
 
 	"github.com/hasura/ndc-http/connector/internal"
 	"github.com/hasura/ndc-http/ndc-http-schema/configuration"
+	rest "github.com/hasura/ndc-http/ndc-http-schema/schema"
 	"github.com/hasura/ndc-sdk-go/connector"
 	"github.com/hasura/ndc-sdk-go/schema"
 )
 
 // HTTPConnector implements the SDK interface of NDC specification
 type HTTPConnector struct {
-	config       *configuration.Configuration
-	metadata     internal.MetadataCollection
-	capabilities *schema.RawCapabilitiesResponse
-	rawSchema    *schema.RawSchemaResponse
-	httpClient   *http.Client
-	upstreams    *internal.UpstreamManager
+	config              *configuration.Configuration
+	metadata            internal.MetadataCollection
+	capabilities        *schema.RawCapabilitiesResponse
+	rawSchema           *schema.RawSchemaResponse
+	httpClient          *http.Client
+	upstreams           *internal.UpstreamManager
+	procSendHttpRequest rest.OperationInfo
 }
 
 // NewHTTPConnector creates a HTTP connector instance

connector/internal/client.go

@@ -292,6 +292,12 @@ func (client *HTTPClient) doRequest(ctx context.Context, request *RetryableReque
 		attribute.String("network.protocol.name", "http"),
 	)
 
+	var namespace string
+	if client.requests.Schema != nil && client.requests.Schema.Name != "" {
+		namespace = client.requests.Schema.Name
+		span.SetAttributes(attribute.String("db.namespace", namespace))
+	}
+
 	if request.ContentLength > 0 {
 		span.SetAttributes(attribute.Int64("http.request.body.size", request.ContentLength))
 	}
@@ -301,8 +307,7 @@ func (client *HTTPClient) doRequest(ctx context.Context, request *RetryableReque
 	setHeaderAttributes(span, "http.request.header.", request.Headers)
 
 	client.propagator.Inject(ctx, propagation.HeaderCarrier(request.Headers))
-
-	resp, cancel, err := client.manager.ExecuteRequest(ctx, request, client.requests.Schema.Name)
+	resp, cancel, err := client.manager.ExecuteRequest(ctx, request, namespace)
 	if err != nil {
 		span.SetStatus(codes.Error, "error happened when executing the request")
 		span.RecordError(err)
@@ -371,7 +376,7 @@ func (client *HTTPClient) evalHTTPResponse(ctx context.Context, span trace.Span,
 
 	var result any
 	switch {
-	case strings.HasPrefix(contentType, "text/"):
+	case strings.HasPrefix(contentType, "text/") || strings.HasPrefix(contentType, "image/svg"):
 		respBody, err := io.ReadAll(resp.Body)
 		if err != nil {
 			return nil, nil, schema.NewConnectorError(http.StatusInternalServerError, err.Error(), nil)
@@ -412,13 +417,18 @@ func (client *HTTPClient) evalHTTPResponse(ctx context.Context, span trace.Span,
 			}
 		}
 
-		responseType, extractErr := client.extractResultType(resultType)
-		if extractErr != nil {
-			return nil, nil, extractErr
+		var err error
+		if client.requests.Schema == nil || client.requests.Schema.NDCHttpSchema == nil {
+			err = json.NewDecoder(resp.Body).Decode(&result)
+		} else {
+			responseType, extractErr := client.extractResultType(resultType)
+			if extractErr != nil {
+				return nil, nil, extractErr
+			}
+
+			result, err = contenttype.NewJSONDecoder(client.requests.Schema.NDCHttpSchema).Decode(resp.Body, responseType)
 		}
 
-		var err error
-		result, err = contenttype.NewJSONDecoder(client.requests.Schema.NDCHttpSchema).Decode(resp.Body, responseType)
 		if err != nil {
 			return nil, nil, schema.NewConnectorError(http.StatusInternalServerError, err.Error(), nil)
 		}

connector/internal/contenttype/multipart.go

@@ -57,6 +57,7 @@ func (mfb *MultipartFormEncoder) evalMultipartForm(w *MultipartWriter, bodyInfo
 	if !ok {
 		return nil
 	}
+
 	switch bodyType := bodyInfo.Type.Interface().(type) {
 	case *schema.NullableType:
 		return mfb.evalMultipartForm(w, &rest.ArgumentInfo{
@@ -73,6 +74,7 @@ func (mfb *MultipartFormEncoder) evalMultipartForm(w *MultipartWriter, bodyInfo
 		if !ok {
 			break
 		}
+
 		kind := bodyData.Kind()
 		switch kind {
 		case reflect.Map, reflect.Interface:
@@ -276,3 +278,52 @@ func (mfb *MultipartFormEncoder) evalEncodingHeaders(encHeaders map[string]rest.
 
 	return results, nil
 }
+
+// EncodeArbitrary encodes the unknown data to multipart/form.
+func (c *MultipartFormEncoder) EncodeArbitrary(bodyData any) (*bytes.Reader, string, error) {
+	buffer := new(bytes.Buffer)
+	writer := NewMultipartWriter(buffer)
+
+	reflectValue, ok := utils.UnwrapPointerFromReflectValue(reflect.ValueOf(bodyData))
+	if ok {
+		valueMap, ok := reflectValue.Interface().(map[string]any)
+		if !ok {
+			return nil, "", fmt.Errorf("invalid body for multipart/form, expected object, got: %s", reflectValue.Kind())
+		}
+
+		for key, value := range valueMap {
+			if err := c.evalFormDataReflection(writer, key, reflect.ValueOf(value)); err != nil {
+				return nil, "", fmt.Errorf("invalid body for multipart/form, %s: %w", key, err)
+			}
+		}
+	}
+
+	if err := writer.Close(); err != nil {
+		return nil, "", err
+	}
+
+	reader := bytes.NewReader(buffer.Bytes())
+	buffer.Reset()
+
+	return reader, writer.FormDataContentType(), nil
+}
+
+func (c *MultipartFormEncoder) evalFormDataReflection(w *MultipartWriter, key string, reflectValue reflect.Value) error {
+	reflectValue, ok := utils.UnwrapPointerFromReflectValue(reflectValue)
+	if !ok {
+		return nil
+	}
+
+	kind := reflectValue.Kind()
+	switch kind {
+	case reflect.Map, reflect.Struct, reflect.Array, reflect.Slice:
+		return w.WriteJSON(key, reflectValue.Interface(), http.Header{})
+	default:
+		value, err := StringifySimpleScalar(reflectValue, kind)
+		if err != nil {
+			return err
+		}
+
+		return w.WriteField(key, value, http.Header{})
+	}
+}

connector/internal/contenttype/url_encode.go

@@ -33,28 +33,51 @@ func NewURLParameterEncoder(schema *rest.NDCHttpSchema, contentType string) *URL
 	}
 }
 
-func (c *URLParameterEncoder) Encode(bodyInfo *rest.ArgumentInfo, bodyData any) (io.ReadSeeker, error) {
+func (c *URLParameterEncoder) Encode(bodyInfo *rest.ArgumentInfo, bodyData any) (io.ReadSeeker, int64, error) {
 	queryParams, err := c.EncodeParameterValues(&rest.ObjectField{
 		ObjectField: schema.ObjectField{
 			Type: bodyInfo.Type,
 		},
 		HTTP: bodyInfo.HTTP.Schema,
 	}, reflect.ValueOf(bodyData), []string{"body"})
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 
 	if len(queryParams) == 0 {
-		return nil, nil
+		return nil, 0, nil
 	}
 	q := url.Values{}
 	for _, qp := range queryParams {
 		keys := qp.Keys()
 		EvalQueryParameterURL(&q, "", bodyInfo.HTTP.EncodingObject, keys, qp.Values())
 	}
 	rawQuery := EncodeQueryValues(q, true)
+	result := bytes.NewReader([]byte(rawQuery))
 
-	return bytes.NewReader([]byte(rawQuery)), nil
+	return result, result.Size(), nil
+}
+
+// Encode marshals the arbitrary body to xml bytes.
+func (c *URLParameterEncoder) EncodeArbitrary(bodyData any) (io.ReadSeeker, int64, error) {
+	queryParams, err := c.encodeParameterReflectionValues(reflect.ValueOf(bodyData), []string{"body"})
+	if err != nil {
+		return nil, 0, err
+	}
+
+	if len(queryParams) == 0 {
+		return nil, 0, nil
+	}
+	q := url.Values{}
+	encObject := rest.EncodingObject{}
+	for _, qp := range queryParams {
+		keys := qp.Keys()
+		EvalQueryParameterURL(&q, "", encObject, keys, qp.Values())
+	}
+	rawQuery := EncodeQueryValues(q, true)
+	result := bytes.NewReader([]byte(rawQuery))
+
+	return result, result.Size(), nil
 }
 
 func (c *URLParameterEncoder) EncodeParameterValues(objectField *rest.ObjectField, reflectValue reflect.Value, fieldPaths []string) (ParameterItems, error) {
@@ -382,6 +405,7 @@ func buildParamQueryKey(name string, encObject rest.EncodingObject, keys Keys, v
 	return strings.Join(resultKeys, "")
 }
 
+// EvalQueryParameterURL evaluate the query parameter URL
 func EvalQueryParameterURL(q *url.Values, name string, encObject rest.EncodingObject, keys Keys, values []string) {
 	if len(values) == 0 {
 		return
@@ -411,6 +435,7 @@ func EvalQueryParameterURL(q *url.Values, name string, encObject rest.EncodingOb
 	}
 }
 
+// EncodeQueryValues encode query values to string.
 func EncodeQueryValues(qValues url.Values, allowReserved bool) string {
 	if !allowReserved {
 		return qValues.Encode()
@@ -433,6 +458,7 @@ func EncodeQueryValues(qValues url.Values, allowReserved bool) string {
 	return builder.String()
 }
 
+// SetHeaderParameters set parameters to request headers
 func SetHeaderParameters(header *http.Header, param *rest.RequestParameter, queryParams ParameterItems) {
 	defaultParam := queryParams.FindDefault()
 	// the param is an array

connector/internal/contenttype/url_encode_test.go

@@ -692,7 +692,7 @@ func TestCreateFormURLEncoded(t *testing.T) {
 			assert.NilError(t, json.Unmarshal([]byte(tc.RawArguments), &arguments))
 			argumentInfo := info.Arguments["body"]
 			builder := NewURLParameterEncoder(ndcSchema, rest.ContentTypeFormURLEncoded)
-			buf, err := builder.Encode(&argumentInfo, arguments["body"])
+			buf, _, err := builder.Encode(&argumentInfo, arguments["body"])
 			assert.NilError(t, err)
 			result, err := io.ReadAll(buf)
 			assert.NilError(t, err)

connector/internal/contenttype/xml_decode.go

@@ -44,6 +44,10 @@ func (c *XMLDecoder) Decode(r io.Reader, resultType schema.Type) (any, error) {
 				return nil, fmt.Errorf("failed to decode the xml result: %w", err)
 			}
 
+			if c.schema == nil {
+				return decodeArbitraryXMLBlock(xmlTree), nil
+			}
+
 			result, err := c.evalXMLField(xmlTree, "", rest.ObjectField{
 				ObjectField: schema.ObjectField{
 					Type: resultType,