Benchmark Hive Router Query Planner in Hive Gateway

.github/workflows/benchmark.template.yaml

@@ -31,6 +31,8 @@ on:
       fork:
         type: number
         required: true
+      waitForUrl:
+        type: string
 
 jobs:
   test:
@@ -39,7 +41,7 @@ jobs:
     steps:
       - name: checkout
         uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
-        
+
       - name: setup node
         uses: the-guild-org/shared-config/setup@v1
         with:
@@ -53,7 +55,7 @@ jobs:
           override: true
 
       - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2
-        
+
       - name: run subgraphs
         run: |
           make run-subgraphs &
@@ -69,7 +71,7 @@ jobs:
       - name: configure eqemu
         uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
         with:
-          platforms: 'linux/arm64,linux/amd64'
+          platforms: "linux/arm64,linux/amd64"
 
       - name: configure docker buildx
         uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3
@@ -99,6 +101,7 @@ jobs:
           MEM_LIMIT: ${{ inputs.memoryLimit }}
           CPU_LIMIT: ${{ inputs.cpuLimit }}
           FORK: ${{ inputs.fork }}
+          WAIT_FOR_URL: ${{ inputs.waitForUrl }}
 
       - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
         if: always()

.github/workflows/federation-v1.workflow.yaml

@@ -4,7 +4,8 @@ on:
   workflow_dispatch: {}
 
 concurrency:
-  group: federation-${{ github.ref }}
+  # we group by event name because we dont want to cancel workflow dispatch runs on pr commits
+  group: federation-${{ github.ref }}-${{ github.event_name }}
   cancel-in-progress: true
 
 jobs:
@@ -21,7 +22,7 @@ jobs:
           - apollo-gateway
           - apollo-router
           - hive-gateway
-          - hive-gateway-bun
+          - hive-gateway-router-runtime
           - hive-router
           - cosmo
           - grafbase
@@ -36,6 +37,7 @@ jobs:
       fork: 2
       memoryLimit: 6gb
       mode: constant
+      waitForUrl: ${{ startsWith(matrix.directory, 'hive-gateway') && 'http://localhost:4000/healthcheck' || '' }}
 
   constant-vus-over-time-report:
     needs: constant-vus-over-time
@@ -44,73 +46,73 @@ jobs:
     with:
       scenarioName: constant-vus-over-time
 
-  constant-vus-subgraphs-delay:
-    needs:
-      - decide-runner
-    strategy:
-      fail-fast: false
-      matrix:
-        directory:
-          - apollo-gateway
-          - apollo-router
-          - hive-gateway
-          - hive-gateway-bun
-          - hive-router
-          - cosmo
-          - grafbase
-    uses: ./.github/workflows/benchmark.template.yaml
-    with:
-      gateway: ${{ matrix.directory }}
-      vu: 50
-      time: ${{ github.event_name == 'pull_request' && '30s' || '60s' }}
-      scenarioName: constant-vus-subgraphs-delay
-      runner: ${{ needs.decide-runner.outputs.runner }}
-      cpuLimit: 3
-      fork: 2
-      memoryLimit: 6gb
-      subgraphDelay: 50
-      mode: constant
-
-  constant-vus-subgraphs-delay-report:
-    needs: constant-vus-subgraphs-delay
-    uses: ./.github/workflows/report.template.yaml
-    secrets: inherit
-    with:
-      scenarioName: constant-vus-subgraphs-delay
-
-  constant-vus-subgraphs-delay-resources:
-    needs:
-      - decide-runner
-    strategy:
-      fail-fast: false
-      matrix:
-        directory:
-          - apollo-gateway
-          - apollo-router
-          - hive-gateway
-          - hive-gateway-bun
-          - hive-router
-          - cosmo
-          - grafbase
-    uses: ./.github/workflows/benchmark.template.yaml
-    with:
-      gateway: ${{ matrix.directory }}
-      vu: 50
-      time: ${{ github.event_name == 'pull_request' && '30s' || '60s' }}
-      scenarioName: constant-vus-subgraphs-delay-resources
-      runner: ${{ needs.decide-runner.outputs.runner }}
-      cpuLimit: 4
-      fork: 3
-      memoryLimit: 8gb
-      subgraphDelay: 50
-      mode: constant
-
-  constant-vus-subgraphs-delay-resources-report:
-    needs: constant-vus-subgraphs-delay-resources
-    uses: ./.github/workflows/report.template.yaml
-    secrets: inherit
-    with:
-      scenarioName: constant-vus-subgraphs-delay-resources
+  # TODO: do we need the delay scenarios?
+  # constant-vus-subgraphs-delay:
+  #   needs:
+  #     - decide-runner
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       directory:
+  #         - apollo-gateway
+  #         - apollo-router
+  #         - hive-gateway
+  #         - hive-gateway-router-runtime
+  #         - hive-router
+  #         - cosmo
+  #         - grafbase
+  #   uses: ./.github/workflows/benchmark.template.yaml
+  #   with:
+  #     gateway: ${{ matrix.directory }}
+  #     vu: 50
+  #     time: ${{ github.event_name == 'pull_request' && '30s' || '60s' }}
+  #     scenarioName: constant-vus-subgraphs-delay
+  #     runner: ${{ needs.decide-runner.outputs.runner }}
+  #     cpuLimit: 3
+  #     fork: 2
+  #     memoryLimit: 6gb
+  #     subgraphDelay: 50
+  #     mode: constant
+  #     waitForUrl: ${{ startsWith(matrix.directory, 'hive-gateway') && 'http://localhost:4000/healthcheck' || '' }}
+  # constant-vus-subgraphs-delay-report:
+  #   needs: constant-vus-subgraphs-delay
+  #   uses: ./.github/workflows/report.template.yaml
+  #   secrets: inherit
+  #   with:
+  #     scenarioName: constant-vus-subgraphs-delay
+  # constant-vus-subgraphs-delay-resources:
+  #   needs:
+  #     - decide-runner
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       directory:
+  #         - apollo-gateway
+  #         - apollo-router
+  #         - hive-gateway
+  #         - hive-gateway-router-runtime
+  #         - hive-router
+  #         - cosmo
+  #         - grafbase
+  #   uses: ./.github/workflows/benchmark.template.yaml
+  #   with:
+  #     gateway: ${{ matrix.directory }}
+  #     vu: 50
+  #     time: ${{ github.event_name == 'pull_request' && '30s' || '60s' }}
+  #     scenarioName: constant-vus-subgraphs-delay-resources
+  #     runner: ${{ needs.decide-runner.outputs.runner }}
+  #     cpuLimit: 4
+  #     fork: 3
+  #     memoryLimit: 8gb
+  #     subgraphDelay: 50
+  #     mode: constant
+  #     waitForUrl: ${{ startsWith(matrix.directory, 'hive-gateway') && 'http://localhost:4000/healthcheck' || '' }}
+  # constant-vus-subgraphs-delay-resources-report:
+  #   needs: constant-vus-subgraphs-delay-resources
+  #   uses: ./.github/workflows/report.template.yaml
+  #   secrets: inherit
+  #   with:
+  #     scenarioName: constant-vus-subgraphs-delay-resources
 
   ramping-vus:
     needs:
@@ -122,7 +124,7 @@ jobs:
           - apollo-gateway
           - apollo-router
           - hive-gateway
-          - hive-gateway-bun
+          - hive-gateway-router-runtime
           - hive-router
           - cosmo
           - grafbase
@@ -137,6 +139,7 @@ jobs:
       fork: 3
       memoryLimit: 8gb
       mode: stress
+      waitForUrl: ${{ startsWith(matrix.directory, 'hive-gateway') && 'http://localhost:4000/healthcheck' || '' }}
 
   ramping-vus-report:
     needs: ramping-vus

README.md

@@ -91,7 +91,7 @@ The benchmark uses Unix-specific tools and commands that are not available on Wi
         ```bash
         make test gateway=<gateway_name> mode=<constant|stress>
         ```
-        Replace `<gateway_name>` with one of: `apollo-router`,  `cosmo`, `grafbase`, `hive-gateway`, `hive-gateway-bun`, or `hive-router`.
+        Replace `<gateway_name>` with one of: `apollo-router`,  `cosmo`, `grafbase`, `hive-gateway`, `hive-gateway-bun`, `hive-gateway-router-runtime`, `hive-gateway-router-runtime-bun`, or `hive-router`.
 
         Examples:
         ```bash