diff --git a/conformance/conformance.go b/conformance/conformance.go index d1200627e..406124bdd 100644 --- a/conformance/conformance.go +++ b/conformance/conformance.go @@ -45,7 +45,6 @@ import ( // Import necessary types and utilities from the core Gateway API conformance suite. gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" // Import core Gateway API types confapis "sigs.k8s.io/gateway-api/conformance/apis/v1" // Report struct definition - confconfig "sigs.k8s.io/gateway-api/conformance/utils/config" confflags "sigs.k8s.io/gateway-api/conformance/utils/flags" apikubernetes "sigs.k8s.io/gateway-api/conformance/utils/kubernetes" confsuite "sigs.k8s.io/gateway-api/conformance/utils/suite" @@ -167,7 +166,7 @@ func DefaultOptions(t *testing.T) confsuite.ConformanceOptions { Debug: *confflags.ShowDebug, CleanupBaseResources: *confflags.CleanupBaseResources, SupportedFeatures: sets.New[features.FeatureName](), - TimeoutConfig: confconfig.DefaultTimeoutConfig(), + TimeoutConfig: inferenceconfig.DefaultInferenceExtensionTimeoutConfig().TimeoutConfig, SkipTests: skipTests, ExemptFeatures: exemptFeatures, RunTest: *confflags.RunTest, @@ -177,6 +176,7 @@ func DefaultOptions(t *testing.T) confsuite.ConformanceOptions { ManifestFS: []fs.FS{&Manifests}, ReportOutputPath: *confflags.ReportOutput, SkipProvisionalTests: *confflags.SkipProvisionalTests, + AllowCRDsMismatch: *confflags.AllowCRDsMismatch, NamespaceLabels: namespaceLabels, NamespaceAnnotations: namespaceAnnotations, // TODO: Add the inference extension specific fields to ConformanceOptions struct if needed, diff --git a/conformance/resources/manifests/manifests.yaml b/conformance/resources/manifests/manifests.yaml index c4f9b1f14..a62275f50 100644 --- a/conformance/resources/manifests/manifests.yaml +++ b/conformance/resources/manifests/manifests.yaml @@ -66,6 +66,7 @@ metadata: labels: app: primary-inference-model-server spec: + replicas: 3 selector: matchLabels: app: primary-inference-model-server diff --git a/conformance/tests/basic/gateway_following_epp_routing.go b/conformance/tests/basic/gateway_following_epp_routing.go new file mode 100644 index 000000000..987799977 --- /dev/null +++ b/conformance/tests/basic/gateway_following_epp_routing.go @@ -0,0 +1,194 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package basic + +import ( + "fmt" + "net/http" + "slices" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/gateway-api/conformance/utils/suite" + "sigs.k8s.io/gateway-api/pkg/features" + + "sigs.k8s.io/gateway-api-inference-extension/conformance/tests" + k8sutils "sigs.k8s.io/gateway-api-inference-extension/conformance/utils/kubernetes" + "sigs.k8s.io/gateway-api-inference-extension/conformance/utils/traffic" + trafficutils "sigs.k8s.io/gateway-api-inference-extension/conformance/utils/traffic" + gwhttp "sigs.k8s.io/gateway-api/conformance/utils/http" +) + +func init() { + // Register the GatewayFollowingEPPRouting test case with the conformance suite. + // This ensures it will be discovered and run by the test runner. + tests.ConformanceTests = append(tests.ConformanceTests, GatewayFollowingEPPRouting) +} + +// GatewayFollowingEPPRouting defines the test case for verifying gateway should send traffic to an endpoint in the list returned by EPP. +var GatewayFollowingEPPRouting = suite.ConformanceTest{ + ShortName: "GatewayFollowingEPPRouting", + Description: "Inference gateway should send traffic to an endpoint in the list returned by EPP", + Manifests: []string{"tests/basic/gateway_following_epp_routing.yaml"}, + Features: []features.FeatureName{ + features.FeatureName("SupportInferencePool"), + features.SupportGateway, + }, + Test: func(t *testing.T, s *suite.ConformanceTestSuite) { + const ( + appBackendNamespace = "gateway-conformance-app-backend" + infraNamespace = "gateway-conformance-infra" + hostname = "primary.example.com" + path = "/primary-gateway-test" + expectedPodReplicas = 3 + // eppSelectionHeaderName is the custom header used by the testing-EPP service + // to determine which endpoint to select. + eppSelectionHeaderName = "test-epp-endpoint-selection" + appPodBackendPrefix = "primary-inference-model-server" + ) + + httpRouteNN := types.NamespacedName{Name: "httproute-for-primary-gw", Namespace: appBackendNamespace} + gatewayNN := types.NamespacedName{Name: "conformance-primary-gateway", Namespace: infraNamespace} + poolNN := types.NamespacedName{Name: "primary-inference-pool", Namespace: appBackendNamespace} + backendPodLabels := map[string]string{"app": "primary-inference-model-server"} + + t.Log("Verifying HTTPRoute and InferencePool are accepted and the Gateway has an address.") + k8sutils.HTTPRouteMustBeAcceptedAndResolved(t, s.Client, s.TimeoutConfig, httpRouteNN, gatewayNN) + k8sutils.InferencePoolMustBeAcceptedByParent(t, s.Client, poolNN) + gwAddr := k8sutils.GetGatewayEndpoint(t, s.Client, s.TimeoutConfig, gatewayNN) + + t.Logf("Fetching backend pods with labels: %v", backendPodLabels) + pods, err := k8sutils.GetPodsWithLabel(t, s.Client, appBackendNamespace, backendPodLabels) + require.NoError(t, err, "Failed to get backend pods") + require.Len(t, pods, expectedPodReplicas, "Expected to find %d backend pods, but found %d.", expectedPodReplicas, len(pods)) + + podIPs := make([]string, len(pods)) + podNames := make([]string, len(pods)) + for i, pod := range pods { + podIPs[i] = pod.Status.PodIP + podNames[i] = pod.Name + } + + requestBody := `{ + "model": "conformance-fake-model", + "prompt": "Write as if you were a critic: San Francisco" + }` + + for i := 0; i < len(pods); i++ { + // Send an initial request targeting a single pod and wait for it to be successful to ensure the Gateway and EPP + // are functioning correctly before running the main test cases. + trafficutils.MakeRequestWithRequestParamAndExpectSuccess( + t, + s.RoundTripper, + s.TimeoutConfig, + gwAddr, + trafficutils.Request{ + Host: hostname, + Path: path, + Headers: map[string]string{eppSelectionHeaderName: podIPs[i]}, + Method: http.MethodPost, + Body: requestBody, + Backend: podNames[i], + Namespace: appBackendNamespace, + }, + ) + } + + testCases := []struct { + name string + podIPsToBeReturnedByEPP []string + expectAllRequestsRoutedWithinPodNames []string + }{ + { + name: "should route traffic to a single designated pod", + podIPsToBeReturnedByEPP: []string{podIPs[2]}, + expectAllRequestsRoutedWithinPodNames: []string{podNames[2]}, + }, + { + name: "should route traffic to two designated pods", + podIPsToBeReturnedByEPP: []string{podIPs[0], podIPs[1]}, + expectAllRequestsRoutedWithinPodNames: []string{podNames[0], podNames[1]}, + }, + { + name: "should route traffic to all available pods", + podIPsToBeReturnedByEPP: []string{podIPs[0], podIPs[1], podIPs[2]}, + expectAllRequestsRoutedWithinPodNames: []string{podNames[0], podNames[1], podNames[2]}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + eppHeaderValue := strings.Join(tc.podIPsToBeReturnedByEPP, ",") + headers := map[string]string{eppSelectionHeaderName: eppHeaderValue} + + t.Logf("Sending request to %s with EPP header '%s: %s'", gwAddr, eppSelectionHeaderName, eppHeaderValue) + t.Logf("Expecting traffic to be routed to pod: %v", tc.expectAllRequestsRoutedWithinPodNames) + + assertTrafficOnlyReachesToExpectedPods(t, s, gwAddr, gwhttp.ExpectedResponse{ + Request: gwhttp.Request{ + Host: hostname, + Path: path, + Method: http.MethodPost, + Headers: headers, + }, + Response: gwhttp.Response{ + StatusCode: http.StatusOK, + }, + Backend: appPodBackendPrefix, + Namespace: appBackendNamespace, + }, requestBody, tc.expectAllRequestsRoutedWithinPodNames) + }) + } + }, +} + +func assertTrafficOnlyReachesToExpectedPods(t *testing.T, suite *suite.ConformanceTestSuite, gwAddr string, expected gwhttp.ExpectedResponse, requestBody string, expectedPodNames []string) { + t.Helper() + const ( + concurrentRequests = 10 + totalRequests = 100 + ) + var ( + roundTripper = suite.RoundTripper + g errgroup.Group + req = gwhttp.MakeRequest(t, &expected, gwAddr, "HTTP", "http") + ) + g.SetLimit(concurrentRequests) + for i := 0; i < totalRequests; i++ { + g.Go(func() error { + cReq, cRes, err := traffic.MakeCallRoundTripper(t, roundTripper, &traffic.RequestWithBody{Request: req, Body: strings.NewReader(requestBody)}) + if err != nil { + return fmt.Errorf("failed to roundtrip request: %w", err) + } + if err := gwhttp.CompareRequest(t, &req, cReq, cRes, expected); err != nil { + return fmt.Errorf("response expectation failed for request: %w", err) + } + + if slices.Contains(expectedPodNames, cReq.Pod) { + return nil + } + return fmt.Errorf("request was handled by an unexpected pod %q", cReq.Pod) + }) + } + if err := g.Wait(); err != nil { + t.Fatalf("Not all the requests are sent to the expectedPods successfully, err: %v", err) + } + t.Logf("Traffic successfully reached only to expected pods: %v", expectedPodNames) +} diff --git a/conformance/tests/basic/gateway_following_epp_routing.yaml b/conformance/tests/basic/gateway_following_epp_routing.yaml new file mode 100644 index 000000000..d290b7541 --- /dev/null +++ b/conformance/tests/basic/gateway_following_epp_routing.yaml @@ -0,0 +1,38 @@ +# --- InferenceModel Definition --- +# Service for the infra-backend-deployment. +apiVersion: inference.networking.x-k8s.io/v1alpha2 +kind: InferenceModel +metadata: + name: conformance-fake-model-server + namespace: gateway-conformance-app-backend +spec: + modelName: conformance-fake-model + criticality: Critical # Mark it as critical to bypass the saturation check since the model server is fake and don't have such metrics. + poolRef: + name: primary-inference-pool +--- +# --- HTTPRoute for Primary Gateway (conformance-gateway) --- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: httproute-for-primary-gw + namespace: gateway-conformance-app-backend +spec: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: conformance-primary-gateway + namespace: gateway-conformance-infra + sectionName: http + hostnames: + - "primary.example.com" + rules: + - backendRefs: + - group: inference.networking.x-k8s.io + kind: InferencePool + name: primary-inference-pool + matches: + - path: + type: PathPrefix + value: /primary-gateway-test + \ No newline at end of file diff --git a/conformance/utils/config/timing.go b/conformance/utils/config/timing.go index f520ec702..861eb0f90 100644 --- a/conformance/utils/config/timing.go +++ b/conformance/utils/config/timing.go @@ -20,6 +20,7 @@ import ( "time" // Import the upstream Gateway API timeout config + gatewayconfig "sigs.k8s.io/gateway-api/conformance/utils/config" ) @@ -44,8 +45,12 @@ type InferenceExtensionTimeoutConfig struct { // DefaultInferenceExtensionTimeoutConfig returns a new InferenceExtensionTimeoutConfig with default values. func DefaultInferenceExtensionTimeoutConfig() InferenceExtensionTimeoutConfig { + config := gatewayconfig.DefaultTimeoutConfig() + config.HTTPRouteMustHaveCondition = 300 * time.Second + config.MaxTimeToConsistency = 200 * time.Second + config.DefaultTestTimeout = 600 * time.Second return InferenceExtensionTimeoutConfig{ - TimeoutConfig: gatewayconfig.DefaultTimeoutConfig(), + TimeoutConfig: config, // Initialize embedded struct GeneralMustHaveConditionTimeout: 300 * time.Second, InferencePoolMustHaveConditionInterval: 10 * time.Second, GatewayObjectPollInterval: 5 * time.Second, diff --git a/conformance/utils/kubernetes/helpers.go b/conformance/utils/kubernetes/helpers.go index f5cef755f..8b917c518 100644 --- a/conformance/utils/kubernetes/helpers.go +++ b/conformance/utils/kubernetes/helpers.go @@ -308,6 +308,28 @@ func GetGatewayEndpoint(t *testing.T, k8sClient client.Client, timeoutConfig gat return gwAddr } +// GetPodsWithLabel retrieves a list of Pods. +// It finds pods matching the given labels in a specific namespace. +func GetPodsWithLabel(t *testing.T, c client.Client, namespace string, labels map[string]string) ([]corev1.Pod, error) { + t.Helper() + + podList := &corev1.PodList{} + listOptions := []client.ListOption{ + client.InNamespace(namespace), + client.MatchingLabels(labels), + } + + t.Logf("Searching for Pods with labels %v in namespace %s", labels, namespace) + if err := c.List(context.Background(), podList, listOptions...); err != nil { + return nil, fmt.Errorf("failed to list pods with labels '%v' in namespace '%s': %w", labels, namespace, err) + } + + if len(podList.Items) == 0 { + return nil, fmt.Errorf("no pods found with labels '%v' in namespace '%s'", labels, namespace) + } + return podList.Items, nil +} + // GetPod waits for a Pod matching the specified labels to exist in the given // namespace and have an IP address assigned. This function returns the first // matching Pod found if there are multiple matches. It fails the on timeout or error. diff --git a/conformance/utils/traffic/traffic.go b/conformance/utils/traffic/traffic.go index 07f944919..e27722349 100644 --- a/conformance/utils/traffic/traffic.go +++ b/conformance/utils/traffic/traffic.go @@ -17,16 +17,47 @@ limitations under the License. package traffic import ( + "context" + "encoding/json" "fmt" + "io" "net/http" + "net/http/httputil" + "regexp" + "strings" "testing" + "time" corev1 "k8s.io/api/core/v1" gwconfig "sigs.k8s.io/gateway-api/conformance/utils/config" gwhttp "sigs.k8s.io/gateway-api/conformance/utils/http" "sigs.k8s.io/gateway-api/conformance/utils/roundtripper" + "sigs.k8s.io/gateway-api/conformance/utils/tlog" ) +// Request defines the parameters for a single HTTP test request and its expected outcome. +type Request struct { + // Host is the hostname to use in the HTTP request. + Host string + // Path is the path to request. + Path string + // Method is the HTTP method to use. Defaults to "GET" if empty. + Method string + // Headers are the HTTP headers to include in the request. + Headers map[string]string + // Body is the request body. + Body string + + // ExpectedStatusCode is the HTTP status code expected in the response. + ExpectedStatusCode int + // Backend is the name of the backend service expected to handle the request. + // This is not checked for non-200 responses. + Backend string + // Namespace is the namespace of the backend service. + Namespace string +} + +// Deprecated: this will be replaced by letting caller to construct the Request type. // BuildExpectedHTTPResponse constructs a gwhttp.ExpectedResponse for common test scenarios. // For 200 OK responses, it sets up ExpectedRequest to check Host and Path. // For other status codes (like 404), ExpectedRequest is nil as detailed backend checks are usually skipped by CompareRequest. @@ -62,6 +93,8 @@ func BuildExpectedHTTPResponse( return resp } +// Deprecated: please use MakeRequestWithRequestParamAndExpectSuccess instead. +// https://github.com/kubernetes-sigs/gateway-api-inference-extension/issues/983 // MakeRequestAndExpectSuccess is a helper function that builds an expected success (200 OK) response // and then calls MakeRequestAndExpectEventuallyConsistentResponse. func MakeRequestAndExpectSuccess( @@ -85,6 +118,8 @@ func MakeRequestAndExpectSuccess( gwhttp.MakeRequestAndExpectEventuallyConsistentResponse(t, r, timeoutConfig, gatewayAddress, expectedResponse) } +// Deprecated: please use MakeRequestAndExpectEventuallyConsistentResponse instead and specify the ExpectedStatusCode in Request. +// https://github.com/kubernetes-sigs/gateway-api-inference-extension/issues/983 // MakeRequestAndExpectNotFound is a helper function that builds an expected not found (404) response // and then calls MakeRequestAndExpectEventuallyConsistentResponse. func MakeRequestAndExpectNotFound( @@ -106,6 +141,35 @@ func MakeRequestAndExpectNotFound( gwhttp.MakeRequestAndExpectEventuallyConsistentResponse(t, r, timeoutConfig, gatewayAddress, expectedResponse) } +// MakeRequestAndExpectEventuallyConsistentResponse makes a request using the parameters +// from the Request struct and waits for the response to consistently match the expectations. +func MakeRequestAndExpectEventuallyConsistentResponse( + t *testing.T, + r roundtripper.RoundTripper, + timeoutConfig gwconfig.TimeoutConfig, + gatewayAddress string, + req Request, +) { + t.Helper() + + expectedResponse := makeExpectedResponse(t, req) + waitForConvergeToExpected(t, r, timeoutConfig, gatewayAddress, req.Body, expectedResponse) +} + +// MakeRequestWithRequestParamAndExpectSuccess is a convenience wrapper for requests that are +// expected to succeed with a 200 OK status. +func MakeRequestWithRequestParamAndExpectSuccess( + t *testing.T, + r roundtripper.RoundTripper, + timeoutConfig gwconfig.TimeoutConfig, + gatewayAddress string, + req Request, +) { + t.Helper() + req.ExpectedStatusCode = http.StatusOK + MakeRequestAndExpectEventuallyConsistentResponse(t, r, timeoutConfig, gatewayAddress, req) +} + // MakeRequestAndExpectResponseFromPod sends a request to the specified path by IP address and // uses a special "test-epp-endpoint-selection" header to target a specific backend Pod. // It then verifies that the response was served by that Pod. @@ -130,3 +194,208 @@ func MakeRequestAndExpectResponseFromPod(t *testing.T, r roundtripper.RoundTripp gwhttp.MakeRequestAndExpectEventuallyConsistentResponse(t, r, timeoutConfig, gwAddr, expectedResponse) } + +func makeExpectedResponse(t *testing.T, req Request) gwhttp.ExpectedResponse { + t.Helper() + + method := http.MethodGet + if req.Method != "" { + method = req.Method + } + + expectedResponse := gwhttp.ExpectedResponse{ + Request: gwhttp.Request{ + Host: req.Host, + Path: req.Path, + Method: method, + Headers: req.Headers, + }, + Response: gwhttp.Response{ + StatusCode: req.ExpectedStatusCode, + }, + Backend: req.Backend, + Namespace: req.Namespace, + } + + // For successful responses (200 OK), we also verify that the backend + // received the request with the correct details (Host, Path, etc.). + // For other statuses (e.g., 404), this check is skipped. + if req.ExpectedStatusCode == http.StatusOK { + expectedResponse.ExpectedRequest = &gwhttp.ExpectedRequest{ + Request: gwhttp.Request{ + Host: req.Host, + Path: req.Path, + Headers: req.Headers, + Method: method, + }, + } + } + return expectedResponse +} + +// TODO: https://github.com/kubernetes-sigs/gateway-api-inference-extension/issues/1031 +// replace the following method when sigs.k8s.io/gateway-api/conformance/utils/roundtripper is able to send request with body. +func waitForConvergeToExpected( + t *testing.T, + r roundtripper.RoundTripper, + timeoutConfig gwconfig.TimeoutConfig, + gatewayAddress string, + requestBody string, + expectedResponse gwhttp.ExpectedResponse, +) { + gwhttp.AwaitConvergence(t, timeoutConfig.RequiredConsecutiveSuccesses, timeoutConfig.MaxTimeToConsistency, func(elapsed time.Duration) bool { + req := gwhttp.MakeRequest(t, &expectedResponse, gatewayAddress, "HTTP", "http") + request := &RequestWithBody{Request: req} + if requestBody != "" { + request = &RequestWithBody{Request: req, Body: strings.NewReader(requestBody)} + } + cReq, cRes, err := MakeCallRoundTripper(t, r, request) + if err != nil { + tlog.Logf(t, "Request failed, not ready yet: %v (after %v)", err.Error(), elapsed) + return false + } + + if err := gwhttp.CompareRequest(t, &request.Request, cReq, cRes, expectedResponse); err != nil { + tlog.Logf(t, "Response expectation failed for request: %+v not ready yet: %v (after %v)", request.Request, err, elapsed) + return false + } + + return true + }) + tlog.Logf(t, "Request passed") +} + +// TODO: https://github.com/kubernetes-sigs/gateway-api-inference-extension/issues/1031 +// remove this when sigs.k8s.io/gateway-api/conformance/utils/roundtripper is able to send request with body. +// RequestWithBody extends roundtripper.Request to include a request body. +type RequestWithBody struct { + roundtripper.Request + Body io.Reader +} + +// TODO: https://github.com/kubernetes-sigs/gateway-api-inference-extension/issues/1031 +// remove this when sigs.k8s.io/gateway-api/conformance/utils/roundtripper is able to send request with body. +// MakeCallRoundTripper executes an HTTP request using the provided RoundTripper and captures the request and response. +func MakeCallRoundTripper(t *testing.T, r roundtripper.RoundTripper, request *RequestWithBody) (*roundtripper.CapturedRequest, *roundtripper.CapturedResponse, error) { + client := &http.Client{} + + defaultRoundTripper, ok := r.(*roundtripper.DefaultRoundTripper) + if !ok { + t.Fatalf("Unsupported RoundTripper type: %T", r) + } + rt := defaultRoundTripper + if request.UnfollowRedirect { + client.CheckRedirect = func(_ *http.Request, _ []*http.Request) error { + return http.ErrUseLastResponse + } + } + + client.Transport = &http.Transport{ + DialContext: rt.CustomDialContext, + // We disable keep-alives so that we don't leak established TCP connections. + // Leaking TCP connections is bad because we could eventually hit the + // threshold of maximum number of open TCP connections to a specific + // destination. Keep-alives are not presently utilized so disabling this has + // no adverse affect. + // + // Ref. https://github.com/kubernetes-sigs/gateway-api/issues/2357 + DisableKeepAlives: true, + } + + method := "GET" + if request.Method != "" { + method = request.Method + } + ctx, cancel := context.WithTimeout(context.Background(), rt.TimeoutConfig.RequestTimeout) + defer cancel() + req, err := http.NewRequestWithContext(ctx, method, request.URL.String(), request.Body) + if err != nil { + return nil, nil, err + } + + if request.Host != "" { + req.Host = request.Host + } + + if request.Headers != nil { + for name, value := range request.Headers { + req.Header.Set(name, value[0]) + } + } + + if rt.Debug { + var dump []byte + dump, err = httputil.DumpRequestOut(req, true) + if err != nil { + return nil, nil, err + } + + tlog.Logf(request.T, "Sending Request:\n%s\n\n", formatDump(dump, "< ")) + } + + resp, err := client.Do(req) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + + if rt.Debug { + var dump []byte + dump, err = httputil.DumpResponse(resp, true) + if err != nil { + return nil, nil, err + } + + tlog.Logf(request.T, "Received Response:\n%s\n\n", formatDump(dump, "< ")) + } + + cReq := &roundtripper.CapturedRequest{} + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, nil, err + } + + // we cannot assume the response is JSON + if resp.Header.Get("Content-type") == "application/json" { + err = json.Unmarshal(body, cReq) + if err != nil { + return nil, nil, fmt.Errorf("unexpected error reading response: %w", err) + } + } else { + cReq.Method = method // assume it made the right request if the service being called isn't echoing + } + + cRes := &roundtripper.CapturedResponse{ + StatusCode: resp.StatusCode, + ContentLength: resp.ContentLength, + Protocol: resp.Proto, + Headers: resp.Header, + } + + if resp.TLS != nil { + cRes.PeerCertificates = resp.TLS.PeerCertificates + } + + if roundtripper.IsRedirect(resp.StatusCode) { + redirectURL, err := resp.Location() + if err != nil { + return nil, nil, err + } + cRes.RedirectRequest = &roundtripper.RedirectRequest{ + Scheme: redirectURL.Scheme, + Host: redirectURL.Hostname(), + Port: redirectURL.Port(), + Path: redirectURL.Path, + } + } + + return cReq, cRes, nil +} + +var startLineRegex = regexp.MustCompile(`(?m)^`) + +func formatDump(data []byte, prefix string) string { + data = startLineRegex.ReplaceAllLiteral(data, []byte(prefix)) + return string(data) +}