Skip to content

Unify e2e yardstick backend on 1.2.0 #6004

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JAORMX merged 1 commit into from
Jul 27, 2026
Merged

Unify e2e yardstick backend on 1.2.0 #6004

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .github/workflows/e2e-tests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,12 +130,12 @@ jobs:
docker pull ghcr.io/stackloklabs/osv-mcp/server:0.1.3 &
docker pull ghcr.io/stackloklabs/gofetch/server:1.0.2 &
docker pull ghcr.io/stacklok/toolhive/egress-proxy:latest &
# yardstick (Legacy 1.1.1) is the backend for the vmcp test suite
# yardstick is only needed for the vmcp test suite
if [ "${{ matrix.label_filter }}" = "vmcp" ]; then
docker pull ghcr.io/stackloklabs/yardstick/yardstick-server:1.1.1 &
docker pull ghcr.io/stackloklabs/yardstick/yardstick-server:1.2.0 &
fi
# the time server and the Modern (1.2.0) yardstick (dual-era barrier mode)
# are only used by the proxy (streamable-http) test suite
# the time server and yardstick (dual-era barrier mode) are only
# used by the proxy (streamable-http) test suite
if [ "${{ matrix.label_filter }}" = "proxy && !isolation" ]; then
docker pull ghcr.io/stacklok/dockyard/uvx/mcp-server-time:2026.1.26 &
docker pull ghcr.io/stackloklabs/yardstick/yardstick-server:1.2.0 &
Expand Down
7 changes: 1 addition & 6 deletions .github/workflows/test-e2e-lifecycle.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,7 @@ jobs:
runs-on: ubuntu-8cores-32gb
timeout-minutes: 30
env:
# Legacy (go-sdk v1.6.1) backend for the operator + virtualmcp suites.
YARDSTICK_IMAGE: ghcr.io/stackloklabs/yardstick/yardstick-server:1.1.1
# Modern (go-sdk v1.7) backend used only by the dual-era k8s spec (#5837).
YARDSTICK_IMAGE_DUAL_ERA: ghcr.io/stackloklabs/yardstick/yardstick-server:1.2.0
YARDSTICK_IMAGE: ghcr.io/stackloklabs/yardstick/yardstick-server:1.2.0
defaults:
run:
shell: bash
Expand Down Expand Up @@ -99,7 +96,6 @@ jobs:
# Pull and load all test server images in parallel to speed up CI
echo "Pulling and loading test server images..."
docker pull ${{ env.YARDSTICK_IMAGE }} &
docker pull ${{ env.YARDSTICK_IMAGE_DUAL_ERA }} &
docker pull ghcr.io/stackloklabs/gofetch/server:1.0.1 &
docker pull ghcr.io/stackloklabs/osv-mcp/server:0.0.7 &
docker pull python:3.9-slim &
Expand All @@ -110,7 +106,6 @@ jobs:

# Load all images into kind
kind load docker-image --name toolhive ${{ env.YARDSTICK_IMAGE }}
kind load docker-image --name toolhive ${{ env.YARDSTICK_IMAGE_DUAL_ERA }}
kind load docker-image --name toolhive ghcr.io/stackloklabs/gofetch/server:1.0.1
kind load docker-image --name toolhive ghcr.io/stackloklabs/osv-mcp/server:0.0.7
kind load docker-image --name toolhive python:3.9-slim
Expand Down
8 changes: 4 additions & 4 deletions test/e2e/dual_era_functional_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,10 +41,10 @@ import (
// the suite's global timeout.
const execTimeout = 90 * time.Second

// yardstickClientVersion is derived from images.YardstickServerImageDualEra's tag
// yardstickClientVersion is derived from images.YardstickServerImage's tag
// (the part after ":") so a future image bump can't silently pair a
// mismatched go-sdk version between yardstick-server and yardstick-client.
var yardstickClientVersion = "v" + strings.SplitN(images.YardstickServerImageDualEra, ":", 2)[1]
var yardstickClientVersion = "v" + strings.SplitN(images.YardstickServerImage, ":", 2)[1]

var (
yardstickClientOnce sync.Once
Expand Down Expand Up @@ -137,7 +137,7 @@ var _ = Describe("Modern Functional Round-Trip — Streamable Proxy", Label("pro
"--name", serverName,
"--transport", "stdio",
"-e", "BACKEND_MODE=echo",
images.YardstickServerImageDualEra,
images.YardstickServerImage,
).ExpectSuccess()

err = e2e.WaitForMCPServer(config, serverName, 60*time.Second)
Expand Down Expand Up @@ -382,7 +382,7 @@ var _ = Describe("server/discover Authorization Guard", Label("proxy", "stateles
"--transport", "stdio",
"-e", "BACKEND_MODE=echo",
"--authz-config", authzConfigPath,
images.YardstickServerImageDualEra,
images.YardstickServerImage,
).ExpectSuccess()

err = e2e.WaitForMCPServer(config, serverName, 60*time.Second)
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/dual_era_mixing_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ var _ = Describe("Dual-Era Concurrent Mixing", Label("proxy", "stateless", "dual
"--name", serverName,
"--transport", "stdio",
"-e", "BACKEND_MODE=echo",
images.YardstickServerImageDualEra,
images.YardstickServerImage,
).ExpectSuccess()

err = e2e.WaitForMCPServer(config, serverName, 60*time.Second)
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/dual_era_observability_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ var _ = Describe("Dual-Era Observability — OTLP Spans", Label("proxy", "statel
"--otel-endpoint", collector.URL(),
"--otel-insecure",
"--otel-sampling-rate", "1.0", // always-sample: assertions must not race the default 10% ratio
images.YardstickServerImageDualEra,
images.YardstickServerImage,
).ExpectSuccess()

err = e2e.WaitForMCPServer(config, serverName, 60*time.Second)
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/dual_era_proxy_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ var _ = Describe("Dual-Era Cross-Delivery Proxy", Label("proxy", "stateless", "d
"-e", "BACKEND_MODE=barrier",
"-e", fmt.Sprintf("BARRIER_N=%d", barrierN),
"-e", fmt.Sprintf("BARRIER_TIMEOUT_SECONDS=%d", int(barrierTimeout.Seconds())),
images.YardstickServerImageDualEra,
images.YardstickServerImage,
).ExpectSuccess()

err = e2e.WaitForMCPServer(config, serverName, 60*time.Second)
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/dual_era_resilience_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ var _ = Describe("Dual-Era Resilience — Fault Injection", Label("proxy", "stat
for _, e := range extraEnv {
args = append(args, "-e", e)
}
args = append(args, images.YardstickServerImageDualEra)
args = append(args, images.YardstickServerImage)

// Bound the request timeout so a hang fails fast in the test, not
// after the streamable proxy's 60s default.
Expand Down
20 changes: 7 additions & 13 deletions test/e2e/images/images.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,24 +12,18 @@ package images

const (
yardstickServerImageURL = "ghcr.io/stackloklabs/yardstick/yardstick-server"
yardstickServerImageTag = "1.1.1"
// YardstickServerImage is the Legacy (2025-11-25, go-sdk v1.6.1) yardstick backend
// used by the operator and vMCP e2e tests across multiple transport protocols
// (stdio, SSE, streamable-http) and tenancy modes.
yardstickServerImageTag = "1.2.0"
// YardstickServerImage (go-sdk v1.7) is the yardstick backend used across the
// operator, vMCP, and dual-era e2e tests, over multiple transport protocols (stdio,
// SSE, streamable-http) and tenancy modes. A session-mode deployment negotiates down
// to the Legacy (2025-11-25) data plane and the vMCP classifies it Legacy from
// server/discover's supportedVersions (#5993), so this single image serves both
// eras -- the dual-era tests drive it Modern (stateless), the rest drive it Legacy.
// Note: This image is also referenced in 8 YAML fixture files under
// test/e2e/chainsaw/operator/. Those files are declarative Kubernetes manifests
// and cannot import Go constants directly.
YardstickServerImage = yardstickServerImageURL + ":" + yardstickServerImageTag

yardstickServerDualEraImageTag = "1.2.0"
// YardstickServerImageDualEra is the Modern-capable (2026-07-28, go-sdk v1.7)
// yardstick backend, used ONLY by the dual-era transport-proxy tests (#5837), which
// drive it as a Modern/stateless backend. The operator/vMCP suites must stay on the
// Legacy YardstickServerImage until the vMCP can drive a Modern backend (#5993): a
// v1.7 server in session mode answers server/discover but rejects the Modern
// data-plane, so the vMCP classifies it Modern and then cannot aggregate it.
YardstickServerImageDualEra = yardstickServerImageURL + ":" + yardstickServerDualEraImageTag

gofetchServerImageURL = "ghcr.io/stackloklabs/gofetch/server"
gofetchServerImageTag = "1.0.1"
// GofetchServerImage is used for testing virtual MCP server features, including
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/thv-operator/acceptance_tests/dual_era_k8s_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ var _ = Describe("Dual-Era Multi-Replica Backend", Ordered, func() {
Namespace: testNamespace,
},
Spec: mcpv1beta1.MCPServerSpec{
Image: images.YardstickServerImageDualEra,
Image: images.YardstickServerImage,
Transport: "streamable-http",
ProxyPort: 8080,
MCPPort: 8080,
Expand Down
Loading