Add GHA coordinator for performance evaluation task scatter/gather

.github/workflows/ec2-leg.yml

@@ -0,0 +1,264 @@
+name: Perf-eval EC2 leg (ephemeral)
+# Generic callable leg: launches an ephemeral EC2 box, renders its user-data from
+# bootstrap-common.sh + the leg's own run script, polls S3 for the result object
+# the box publishes, and reports through that object (job result = leg verdict).
+# Does not post to the PR -- the coordinator owns human-facing output.
+# Box bootstrap: perf-eval/bootstrap-common.sh + the leg script (inputs.leg_script);
+# runner-side polling: perf-eval/gather.
+
+on:
+  # workflow_call is the only trigger: a leg runs through the coordinator.
+  workflow_call:
+    inputs:
+      target_ref:
+        description: 'Ref or SHA to benchmark (resolved to a SHA on checkout)'
+        required: true
+        type: string
+      run_label:
+        description: 'Leg label; folded into the S3 result key and instance name'
+        required: true
+        type: string
+      leg_script:
+        description: 'Path to the leg run script, concatenated after bootstrap-common.sh into the box user-data'
+        required: true
+        type: string
+      instance_type:
+        description: 'EC2 instance type'
+        required: false
+        type: string
+        default: c5.2xlarge
+      root_volume_gb:
+        description: 'Root gp3 volume size (GB)'
+        required: false
+        type: number
+        default: 500
+      volume_iops:
+        description: 'Root gp3 IOPS (3000 is the gp3 floor)'
+        required: false
+        type: number
+        default: 3000
+      volume_throughput:
+        description: 'Root gp3 throughput (MiB/s)'
+        required: false
+        type: number
+        default: 125
+      ssm_registration_timeout:
+        description: 'Seconds to wait for the SSM agent to register'
+        required: false
+        type: number
+        default: 240
+      budget_minutes:
+        description: 'End-to-end leg ceiling (= job timeout); S3 poll deadline, box self-terminate, and OIDC session derive from it (-15m/+15m/+15m). Derived session must fit the IAM role max.'
+        required: false
+        type: number
+        default: 225
+      poll_interval:
+        description: 'Seconds between S3 result polls'
+        required: false
+        type: number
+        default: 30
+      extra_env:
+        description: 'Extra `export FOO=bar` lines injected into the box user-data preamble'
+        required: false
+        type: string
+        default: ''
+    secrets:
+      AWS_GHA_ROLE_ARN:
+        description: 'OIDC role ARN to assume for EC2 + S3 access'
+        required: true
+
+# Only the intrinsic infra permissions necessary
+permissions:
+  id-token: write   # OIDC AssumeRole into the GHA role
+  contents: read    # checkout
+
+jobs:
+  ec2-leg:
+    name: Launch + await ephemeral perf-eval box
+    # Non-cancelling group keyed on the test commit SHA + leg. An instance can run
+    # until the box’s self-terminate ceiling, so new runs queue.
+    concurrency:
+      group: ec2-leg-${{ inputs.target_ref }}-${{ inputs.run_label }}
+      cancel-in-progress: false
+    runs-on: ubuntu-latest
+    timeout-minutes: ${{ inputs.budget_minutes }}
+    env:
+      AWS_REGION: us-east-1
+      INSTANCE_TYPE: ${{ inputs.instance_type }}
+      ROOT_VOLUME_GB: ${{ inputs.root_volume_gb }}
+      BOOTSTRAP_VOLUME_IOPS: ${{ inputs.volume_iops }}
+      BOOTSTRAP_VOLUME_THROUGHPUT: ${{ inputs.volume_throughput }}
+      INSTANCE_PROFILE: stellar-rpc-ci-load-test
+      TEST_TAG_KEY: test
+      TEST_TAG_VAL: stellar-rpc-ci-load-test
+      SSM_REGISTRATION_TIMEOUT: ${{ inputs.ssm_registration_timeout }}
+      POLL_INTERVAL: ${{ inputs.poll_interval }}
+      DEBUG_LOG_LINES: 40
+      DEBUG_LOG_EVERY_POLLS: 5
+      PERF_EVAL_DIR: cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval
+      LEG_SCRIPT: ${{ inputs.leg_script }}
+      BUCKET: stellar-rpc-ci-load-test
+      RESULT_KEY: runs/${{ github.run_id }}/${{ inputs.run_label }}/result.json # deterministic per run + leg
+
+    steps:
+      # budget_minutes is the job timeout and the other deadlines derive from it so
+      # the ordering (poll < job < self-terminate <= role session) holds by
+      # construction.
+      - name: Derive deadlines
+        id: deadlines
+        run: |
+          B=${{ inputs.budget_minutes }}
+          [ "$B" -gt 30 ] || { echo "::error::budget_minutes must exceed the 15m infra buffers"; exit 1; }
+          {
+            echo "RESULTS_TIMEOUT=$(( (B - 15) * 60 ))"
+            echo "SELF_TERMINATE_MINUTES=$(( B + 15 ))"
+          } >> "$GITHUB_ENV"
+          echo "role_seconds=$(( (B + 15) * 60 ))" >> "$GITHUB_OUTPUT"
+
+      # Harness (gather, bootstrap-common.sh, leg script) and the benchmarked
+      # code come from one SHA: target_ref is checked out here and the box rebuilds
+      # it. Tradeoff: an ad-hoc dispatch against an old ref also runs that ref's
+      # harness.
+      - name: Checkout target ref
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.target_ref }}
+
+      - name: Resolve target SHA
+        id: resolve
+        run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+
+      # The runner-side half is `go run <perf-eval>/gather`.
+      - uses: ./.github/actions/setup-go
+
+      - name: Configure AWS via OIDC
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_GHA_ROLE_ARN }}
+          aws-region: ${{ env.AWS_REGION }}
+          role-duration-seconds: ${{ steps.deadlines.outputs.role_seconds }}
+
+      - name: Render user-data
+        # The scripts ship verbatim; parameters travel in a preamble so the bytes
+        # that run on the box match the bytes in git. Order is:
+        #   preamble exports -> bootstrap-common.sh -> the leg run script.
+        env:
+          EXTRA_ENV: ${{ inputs.extra_env }}
+        run: |
+          {
+            echo '#!/usr/bin/env bash'
+            echo "export TARGET_SHA=${{ steps.resolve.outputs.sha }} RUN_ID=${{ github.run_id }}-${{ github.run_attempt }}"
+            echo "export BUCKET=$BUCKET RESULT_KEY=$RESULT_KEY"
+            echo "export SELF_TERMINATE_MINUTES=$SELF_TERMINATE_MINUTES"
+            if [ -n "$EXTRA_ENV" ]; then printf '%s\n' "$EXTRA_ENV"; fi
+            cat "$PERF_EVAL_DIR/bootstrap-common.sh"
+            cat "$LEG_SCRIPT"
+          } > /tmp/user-data.sh
+
+      - name: Launch EC2 instance
+        id: launch
+        env:
+          TARGET_REF: ${{ inputs.target_ref }}
+          RUN_LABEL: ${{ inputs.run_label }}
+        run: |
+          COMMON_TAGS="{Key=$TEST_TAG_KEY,Value=$TEST_TAG_VAL},
+            {Key=ref,Value=$TARGET_REF},
+            {Key=sha,Value=${{ steps.resolve.outputs.sha }}},
+            {Key=run-id,Value=${{ github.run_id }}}"
+          # AMI: Ubuntu 22.04 amd64 in us-east-1 as of 2026-07-08
+          RUN_INSTANCES_JSON=$(aws ec2 run-instances \
+            --image-id ami-0d28727121d5d4a3c \
+            --instance-type "$INSTANCE_TYPE" \
+            --iam-instance-profile "Name=$INSTANCE_PROFILE" \
+            --user-data file:///tmp/user-data.sh \
+            --instance-initiated-shutdown-behavior terminate \
+            --block-device-mappings "[{
+              \"DeviceName\":\"/dev/sda1\",
+              \"Ebs\":{\"VolumeSize\":$ROOT_VOLUME_GB,\"VolumeType\":\"gp3\",\"Iops\":$BOOTSTRAP_VOLUME_IOPS,\"Throughput\":$BOOTSTRAP_VOLUME_THROUGHPUT,\"DeleteOnTermination\":true}
+            }]" \
+            --tag-specifications \
+              "ResourceType=instance,Tags=[
+                {Key=Name,Value=perf-eval-$RUN_LABEL},
+                $COMMON_TAGS
+              ]" \
+              "ResourceType=volume,Tags=[
+                {Key=Name,Value=perf-eval-${RUN_LABEL}-root},
+                $COMMON_TAGS
+              ]" \
+            --count 1 \
+            --output json)
+
+          INSTANCE_ID=$(printf '%s' "$RUN_INSTANCES_JSON" | jq -r '.Instances[0].InstanceId')
+          echo "instance_id=$INSTANCE_ID" >> "$GITHUB_OUTPUT"
+
+      - name: Wait for SSM agent to register
+        env:
+          INSTANCE_ID: ${{ steps.launch.outputs.instance_id }}
+        run: |
+          DEADLINE=$(( $(date +%s) + SSM_REGISTRATION_TIMEOUT ))
+          while [ "$(date +%s)" -lt "$DEADLINE" ]; do
+            PING=$(aws ssm describe-instance-information \
+                     --filters "Key=InstanceIds,Values=$INSTANCE_ID" \
+                     --query 'InstanceInformationList[0].PingStatus' \
+                     --output text 2>/dev/null || echo "")
+            echo "[$(date -u +%FT%TZ)] ssm ping=$PING"
+            if [ "$PING" = "Online" ]; then
+              exit 0
+            fi
+            sleep 10
+          done
+          echo "::error::SSM agent never registered for $INSTANCE_ID — verify AmazonSSMManagedInstanceCore is attached to the stellar-rpc-ci-load-test role"
+          exit 1
+
+      # Polls S3 for the result object and relays verdict/passed + the result
+      # location (bucket/key) as step outputs the coordinator reads.
+      # RUN_ID must match the box preamble's: gather skips result objects from
+      # other attempts (a re-run shares RESULT_KEY with its predecessors).
+      - name: Poll for results
+        id: results
+        env:
+          INSTANCE_ID: ${{ steps.launch.outputs.instance_id }}
+          RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }}
+        run: go run "./$PERF_EVAL_DIR/gather"
+
+      - name: Write results summary
+        if: always()
+        run: |
+          if [ -f /tmp/results.md ]; then
+            cat /tmp/results.md >> "$GITHUB_STEP_SUMMARY"
+          elif [ -f /tmp/timeout-comment.md ]; then
+            cat /tmp/timeout-comment.md >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+      # Standalone-debugging convenience: the result object also lives in S3, which
+      # is what the coordinator reads.
+      - name: Upload leg results artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: perf-eval-results-${{ inputs.run_label }}
+          path: |
+            /tmp/results.md
+            /tmp/timeout-comment.md
+            /tmp/bench-results.json
+          if-no-files-found: warn
+
+      - name: Fail leg on timeout or leg failure
+        if: always()
+        run: |
+          if [ "${{ steps.results.outputs.found }}" != "true" ]; then
+            echo "Leg timed out before producing instance results"
+            exit 1
+          fi
+
+          if [ "${{ steps.results.outputs.passed }}" != "true" ]; then
+            echo "Instance reported a failing verdict"
+            cat /tmp/results.md 2>/dev/null || true
+            exit 1
+          fi
+
+      - name: Terminate instance
+        if: always() && steps.launch.outputs.instance_id != ''
+        run: |
+          aws ec2 terminate-instances \
+            --instance-ids ${{ steps.launch.outputs.instance_id }} || true