From b20af67bf49247da18e07dcd2642215d009ff56b Mon Sep 17 00:00:00 2001
From: Marcus Mann <mpmann@amazon.com>
Date: Fri, 13 Feb 2026 23:04:33 +0000
Subject: [PATCH 01/19] feat(ci): Add Go build cache warming for integration
 tests
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add WarmTestCache workflow job that pre-compiles all test packages and
uploads GOCACHE/GOMODCACHE to S3. EC2 integration test instances
download this cache before running go test, eliminating redundant
compilation.

Changes:
- Add .github/workflows/warm-test-cache.yml reusable workflow
- Add WarmTestCache job to test-artifacts.yml (runs in parallel with
  GenerateTestMatrix)
- Add cache_key input to ec2-integration-test.yml reusable workflow
- Pass cache_key to terraform for EC2 Linux, OnPrem, and SELinux tests

The cache is keyed by branch, Go version, OS, and architecture. It
runs once per workflow execution and all EC2 test jobs consume it.

Companion PR: https://github.com/aws/amazon-cloudwatch-agent-test/pull/650

🤖 Assisted by AI
---
 .github/workflows/ec2-integration-test.yml |  4 ++
 .github/workflows/test-artifacts.yml       | 35 +++++++--
 .github/workflows/warm-test-cache.yml      | 84 ++++++++++++++++++++++
 3 files changed, 116 insertions(+), 7 deletions(-)
 create mode 100644 .github/workflows/warm-test-cache.yml

diff --git a/.github/workflows/ec2-integration-test.yml b/.github/workflows/ec2-integration-test.yml
index db45a9a497..930793edbf 100644
--- a/.github/workflows/ec2-integration-test.yml
+++ b/.github/workflows/ec2-integration-test.yml
@@ -44,6 +44,9 @@ on:
         type: boolean
       s3_integration_bucket:
         type: string
+      cache_key:
+        type: string
+        required: false
     secrets:
       AWS_PRIVATE_KEY:
         required: false
@@ -128,6 +131,7 @@ jobs:
               -var="test_dir=${{ matrix.arrays.test_dir }}" \
               -var="test_name=${{ matrix.arrays.os }}" \
               -var="is_onprem=${{ inputs.is_onprem_test }}" \
+              -var="cache_key=${{ inputs.cache_key }}" \
               -var="user=${{ matrix.arrays.username }}"; then
               terraform destroy -var="region=${{ inputs.region }}" -var="ami=${{ matrix.arrays.ami }}" -auto-approve
             else
diff --git a/.github/workflows/test-artifacts.yml b/.github/workflows/test-artifacts.yml
index ff4c70a63d..7f5826a1da 100644
--- a/.github/workflows/test-artifacts.yml
+++ b/.github/workflows/test-artifacts.yml
@@ -284,6 +284,20 @@ jobs:
           echo "ec2_linux_itar_matrix: ${{ steps.set-matrix.outputs.ec2_linux_itar_matrix }}"
           echo "ec2_linux_china_matrix: ${{ steps.set-matrix.outputs.ec2_linux_china_matrix }}"
 
+  WarmTestCache:
+    name: 'WarmTestCache'
+    needs: [OutputEnvVariables]
+    uses: ./.github/workflows/warm-test-cache.yml
+    secrets: inherit
+    permissions:
+      id-token: write
+      contents: read
+    with:
+      test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }}
+      test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}
+      s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
+      terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
+
   CloudformationTest:
     needs: [OutputEnvVariables, GenerateTestMatrix]
     name: 'CFTest'
@@ -540,7 +554,7 @@ jobs:
             terraform destroy --auto-approve
 
   EC2LinuxIntegrationTest-0:
-    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables ]
+    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, WarmTestCache ]
     name: 'EC2Linux-0'
     uses:  ./.github/workflows/ec2-integration-test.yml
     with:
@@ -555,12 +569,13 @@ jobs:
       region: us-west-2
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
+      cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
       is_selinux_test: false
       is_onprem_test: false
     secrets: inherit
 
   EC2LinuxIntegrationTest-1:
-    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables ]
+    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, WarmTestCache ]
     if: ${{ needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 1 }}
     name: 'EC2Linux-1'
     uses:  ./.github/workflows/ec2-integration-test.yml
@@ -576,12 +591,13 @@ jobs:
       region: us-west-2
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
+      cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
       is_selinux_test: false
       is_onprem_test: false
     secrets: inherit
 
   EC2LinuxIntegrationTest-2:
-    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables ]
+    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, WarmTestCache ]
     if: ${{ needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 2 }}
     name: 'EC2Linux-2'
     uses:  ./.github/workflows/ec2-integration-test.yml
@@ -597,12 +613,13 @@ jobs:
       region: us-west-2
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
+      cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
       is_selinux_test: false
       is_onprem_test: false
     secrets: inherit
 
   EC2LinuxIntegrationTest-3:
-    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables ]
+    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, WarmTestCache ]
     if: ${{ needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 3 }}
     name: 'EC2Linux-3'
     uses:  ./.github/workflows/ec2-integration-test.yml
@@ -618,12 +635,13 @@ jobs:
       region: us-west-2
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
+      cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
       is_selinux_test: false
       is_onprem_test: false
     secrets: inherit
 
   EC2LinuxIntegrationTest-4:
-    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables ]
+    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, WarmTestCache ]
     if: ${{ needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 4 }}
     name: 'EC2Linux-4'
     uses:  ./.github/workflows/ec2-integration-test.yml
@@ -639,12 +657,13 @@ jobs:
       region: us-west-2
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
+      cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
       is_selinux_test: false
       is_onprem_test: false
     secrets: inherit
 
   LinuxOnPremIntegrationTest:
-    needs: [GenerateTestMatrix, OutputEnvVariables]
+    needs: [GenerateTestMatrix, OutputEnvVariables, WarmTestCache]
     name: 'OnpremLinux'
     uses: ./.github/workflows/ec2-integration-test.yml
     with:
@@ -659,6 +678,7 @@ jobs:
       region: us-west-2
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
+      cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
       is_selinux_test: false
       is_onprem_test: true
     secrets: inherit
@@ -705,7 +725,7 @@ jobs:
       KEY_NAME: ${{ secrets.KEY_NAME }}
 
   EC2SELinuxIntegrationTest:
-    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables ]
+    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, WarmTestCache ]
     name: 'EC2SELinux'
     uses:  ./.github/workflows/ec2-integration-test.yml
     with:
@@ -720,6 +740,7 @@ jobs:
       region: us-west-2
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
+      cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
       is_selinux_test: true
 
   EC2WinIntegrationTest:
diff --git a/.github/workflows/warm-test-cache.yml b/.github/workflows/warm-test-cache.yml
new file mode 100644
index 0000000000..0d61e0d1e5
--- /dev/null
+++ b/.github/workflows/warm-test-cache.yml
@@ -0,0 +1,84 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: MIT
+
+name: Warm Go Test Cache
+
+on:
+  workflow_call:
+    inputs:
+      test_repo_name:
+        required: true
+        type: string
+      test_repo_branch:
+        required: true
+        type: string
+      s3_integration_bucket:
+        required: true
+        type: string
+      terraform_assume_role:
+        required: true
+        type: string
+    outputs:
+      cache_key:
+        description: "Cache key for restoring Go build/module caches"
+        value: ${{ jobs.WarmTestCache.outputs.cache_key }}
+
+jobs:
+  WarmTestCache:
+    name: 'WarmTestCache'
+    runs-on: ubuntu-latest
+    outputs:
+      cache_key: ${{ steps.warm.outputs.cache_key }}
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          repository: ${{ inputs.test_repo_name }}
+          ref: ${{ inputs.test_repo_branch }}
+
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v4
+        with:
+          go-version: ~1.25
+          cache: false
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ inputs.terraform_assume_role }}
+          aws-region: us-west-2
+
+      - name: Warm cache and upload to S3
+        id: warm
+        env:
+          S3_BUCKET: ${{ inputs.s3_integration_bucket }}
+          TEST_BRANCH: ${{ inputs.test_repo_branch }}
+        run: |
+          GO_VERSION=$(go env GOVERSION)
+          CACHE_KEY="${TEST_BRANCH}-${GO_VERSION}-linux-amd64"
+          echo "cache_key=${CACHE_KEY}" >> "$GITHUB_OUTPUT"
+          echo "Cache key: ${CACHE_KEY}"
+
+          # Download modules and compile all test packages (without running them)
+          go mod download
+          CGO_ENABLED=0 go test -run='^$' ./test/... 2>/dev/null || true
+
+          # Upload caches to S3
+          GOCACHE=$(go env GOCACHE)
+          GOMODCACHE=$(go env GOMODCACHE)
+          CACHE_PREFIX="s3://${S3_BUCKET}/integration-test/cache/${CACHE_KEY}"
+
+          echo "Compressing build cache from ${GOCACHE}..."
+          tar czf /tmp/gocache.tar.gz -C "$GOCACHE" .
+          echo "Compressing module cache from ${GOMODCACHE}..."
+          tar czf /tmp/gomodcache.tar.gz -C "$GOMODCACHE" .
+
+          echo "Uploading to ${CACHE_PREFIX}..."
+          aws s3 cp /tmp/gocache.tar.gz "${CACHE_PREFIX}/gocache.tar.gz" --quiet
+          aws s3 cp /tmp/gomodcache.tar.gz "${CACHE_PREFIX}/gomodcache.tar.gz" --quiet
+
+          GOCACHE_SIZE=$(du -sh /tmp/gocache.tar.gz | cut -f1)
+          GOMODCACHE_SIZE=$(du -sh /tmp/gomodcache.tar.gz | cut -f1)
+          echo "Build cache: ${GOCACHE_SIZE}, Module cache: ${GOMODCACHE_SIZE}"

From 1c1d8cf92993c14d3c3e1734691a367a83ccd201 Mon Sep 17 00:00:00 2001
From: Marcus Mann <mpmann@amazon.com>
Date: Mon, 16 Feb 2026 19:51:22 +0000
Subject: [PATCH 02/19] feat(ci): Add Go build cache warming to PR-test
 workflow
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Wire WarmTestCache job into PR-test.yml with the same pattern used in
test-artifacts.yml. All EC2 Linux test pages (0-4) and SELinux tests
now receive cache_key and depend on WarmTestCache.

Also added WarmTestCache to verify-all gate check.

🤖 Assisted by AI
---
 .github/workflows/PR-test.yml | 34 +++++++++++++++++++++++++++-------
 1 file changed, 27 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/PR-test.yml b/.github/workflows/PR-test.yml
index 7e4f85f7bf..4f1fa0f01f 100644
--- a/.github/workflows/PR-test.yml
+++ b/.github/workflows/PR-test.yml
@@ -187,10 +187,24 @@ jobs:
           echo "ec2_linux_matrix pages: ${{ steps.paginate-matrix.outputs.ec2_linux_matrix_page_count }}"
           echo "ec2_selinux_matrix: ${{ steps.set-matrix.outputs.ec2_selinux_matrix }}"
 
+  WarmTestCache:
+    name: 'WarmTestCache'
+    needs: [CheckLabel, OutputEnvVariables]
+    if: needs.CheckLabel.outputs.should_run == 'true'
+    uses: ./.github/workflows/warm-test-cache.yml
+    secrets: inherit
+    permissions:
+      id-token: write
+      contents: read
+    with:
+      test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }}
+      test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}
+      s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
+      terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
 
   EC2LinuxIntegrationTest-0:
     name: 'EC2LinuxTests-0'
-    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload ]
+    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload, WarmTestCache ]
     if: needs.CheckLabel.outputs.should_run == 'true'
     uses:  ./.github/workflows/ec2-integration-test.yml
     permissions:
@@ -208,12 +222,13 @@ jobs:
       region: us-west-2
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
+      cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
       is_selinux_test: false
     secrets: inherit
 
   EC2LinuxIntegrationTest-1:
     name: 'EC2LinuxTests-1'
-    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload ]
+    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload, WarmTestCache ]
     if: ${{ needs.CheckLabel.outputs.should_run == 'true' && needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 1 }}
     uses:  ./.github/workflows/ec2-integration-test.yml
     permissions:
@@ -231,12 +246,13 @@ jobs:
       region: us-west-2
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
+      cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
       is_selinux_test: false
     secrets: inherit
 
   EC2LinuxIntegrationTest-2:
     name: 'EC2LinuxTests-2'
-    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload ]
+    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload, WarmTestCache ]
     if: ${{ needs.CheckLabel.outputs.should_run == 'true' && needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 2 }}
     uses:  ./.github/workflows/ec2-integration-test.yml
     permissions:
@@ -254,12 +270,13 @@ jobs:
       region: us-west-2
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
+      cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
       is_selinux_test: false
     secrets: inherit
 
   EC2LinuxIntegrationTest-3:
     name: 'EC2LinuxTests-3'
-    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload ]
+    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload, WarmTestCache ]
     if: ${{ needs.CheckLabel.outputs.should_run == 'true' && needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 3 }}
     uses:  ./.github/workflows/ec2-integration-test.yml
     permissions:
@@ -277,12 +294,13 @@ jobs:
       region: us-west-2
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
+      cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
       is_selinux_test: false
     secrets: inherit
 
   EC2LinuxIntegrationTest-4:
     name: 'EC2LinuxTests-4'
-    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload ]
+    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload, WarmTestCache ]
     if: ${{ needs.CheckLabel.outputs.should_run == 'true' && needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 4 }}
     uses:  ./.github/workflows/ec2-integration-test.yml
     permissions:
@@ -300,12 +318,13 @@ jobs:
       region: us-west-2
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
+      cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
       is_selinux_test: false
     secrets: inherit
 
   EC2SELinuxIntegrationTest:
     name: 'EC2SELinuxTests'
-    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload ]
+    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload, WarmTestCache ]
     if: needs.CheckLabel.outputs.should_run == 'true'
     uses:  ./.github/workflows/ec2-integration-test.yml
     permissions:
@@ -323,6 +342,7 @@ jobs:
       region: us-west-2
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
+      cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
       is_selinux_test: true
     secrets: inherit
 
@@ -364,7 +384,7 @@ jobs:
   verify-all:
     name: Verify All PR Test Jobs
     needs: [IntegrationTestGate, CheckLabel, BuildAndUpload, OutputEnvVariables, StartLocalStack,
-            GenerateTestMatrix, EC2LinuxIntegrationTest-0, EC2LinuxIntegrationTest-1,
+            GenerateTestMatrix, WarmTestCache, EC2LinuxIntegrationTest-0, EC2LinuxIntegrationTest-1,
             EC2LinuxIntegrationTest-2, EC2LinuxIntegrationTest-3, EC2LinuxIntegrationTest-4,
             EC2SELinuxIntegrationTest, StopLocalStack]
     runs-on: ubuntu-latest

From 37d97a2704d30c3d7b91c51b982c0c58f025a6ab Mon Sep 17 00:00:00 2001
From: Marcus Mann <mpmann@amazon.com>
Date: Mon, 16 Feb 2026 19:59:52 +0000
Subject: [PATCH 03/19] fix(ci): Only pass cache_key to terraform when
 non-empty
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Terraform errors with 'undeclared variable' if cache_key is passed to a
terraform config that doesn't declare it. This happens when the test
repo hasn't been updated yet, or for terraform dirs that don't support
caching.

Conditionally build the -var flag so it's only included when cache_key
is set, ensuring backward compatibility.

🤖 Assisted by AI
---
 .github/workflows/ec2-integration-test.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ec2-integration-test.yml b/.github/workflows/ec2-integration-test.yml
index 930793edbf..6cd6009998 100644
--- a/.github/workflows/ec2-integration-test.yml
+++ b/.github/workflows/ec2-integration-test.yml
@@ -106,6 +106,11 @@ jobs:
               cd ${{inputs.test_dir}}
             fi
 
+            CACHE_KEY_VAR=""
+            if [ -n "${{ inputs.cache_key }}" ]; then
+              CACHE_KEY_VAR="-var=cache_key=${{ inputs.cache_key }}"
+            fi
+
             terraform init
             if terraform apply --auto-approve \
               -var="agent_start=${{ matrix.arrays.agentStartCommand }}" \
@@ -131,7 +136,7 @@ jobs:
               -var="test_dir=${{ matrix.arrays.test_dir }}" \
               -var="test_name=${{ matrix.arrays.os }}" \
               -var="is_onprem=${{ inputs.is_onprem_test }}" \
-              -var="cache_key=${{ inputs.cache_key }}" \
+              $CACHE_KEY_VAR \
               -var="user=${{ matrix.arrays.username }}"; then
               terraform destroy -var="region=${{ inputs.region }}" -var="ami=${{ matrix.arrays.ami }}" -auto-approve
             else

From 2cde20a2a94fb9d22b11aa72d911e28051381634 Mon Sep 17 00:00:00 2001
From: Marcus Mann <mpmann@amazon.com>
Date: Mon, 16 Feb 2026 20:08:07 +0000
Subject: [PATCH 04/19] fix(ci): Check terraform config declares cache_key
 before passing it
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The test repo may not have the cache_key variable yet (PR pending).
Check variables.tf for the declaration before passing -var to terraform
to avoid "undeclared variable" errors during the rollout period.

🤖 Assisted by AI
---
 .github/workflows/ec2-integration-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ec2-integration-test.yml b/.github/workflows/ec2-integration-test.yml
index 6cd6009998..20eb5cc01e 100644
--- a/.github/workflows/ec2-integration-test.yml
+++ b/.github/workflows/ec2-integration-test.yml
@@ -107,7 +107,7 @@ jobs:
             fi
 
             CACHE_KEY_VAR=""
-            if [ -n "${{ inputs.cache_key }}" ]; then
+            if [ -n "${{ inputs.cache_key }}" ] && grep -q 'variable "cache_key"' variables.tf 2>/dev/null; then
               CACHE_KEY_VAR="-var=cache_key=${{ inputs.cache_key }}"
             fi
 

From 6333a71737da0a879613cab099313700b1128e30 Mon Sep 17 00:00:00 2001
From: Marcus Mann <mpmann@amazon.com>
Date: Tue, 17 Feb 2026 20:03:26 +0000
Subject: [PATCH 05/19] feat(ci): Add pre-compiled test binary build and
 distribution (Phase 2)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add BuildTestBinaries workflow that compiles all test packages with
go test -c and uploads static binaries to S3. EC2 test instances
download and run these directly, eliminating ~3 min of compilation
per test.

Changes:
- Add .github/workflows/build-test-binaries.yml reusable workflow
- Add BuildTestBinaries job to test-artifacts.yml and PR-test.yml
  (runs in parallel with GenerateTestMatrix and WarmTestCache)
- Add test_binaries_prefix input to ec2-integration-test.yml
- Conditionally pass -var=test_binaries_prefix to terraform

Binaries are cross-compiled with CGO_ENABLED=0 GOOS=linux GOARCH=amd64
for full portability across all Linux distros (AL2, AL2023, Ubuntu,
RHEL, SLES, etc).

🤖 Assisted by AI
---
 .github/workflows/PR-test.yml              | 35 +++++++--
 .github/workflows/build-test-binaries.yml  | 87 ++++++++++++++++++++++
 .github/workflows/ec2-integration-test.yml |  9 +++
 .github/workflows/test-artifacts.yml       | 35 +++++++--
 4 files changed, 152 insertions(+), 14 deletions(-)
 create mode 100644 .github/workflows/build-test-binaries.yml

diff --git a/.github/workflows/PR-test.yml b/.github/workflows/PR-test.yml
index 4f1fa0f01f..712cdbc5a5 100644
--- a/.github/workflows/PR-test.yml
+++ b/.github/workflows/PR-test.yml
@@ -202,9 +202,24 @@ jobs:
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
 
+  BuildTestBinaries:
+    name: 'BuildTestBinaries'
+    needs: [CheckLabel, OutputEnvVariables]
+    if: needs.CheckLabel.outputs.should_run == 'true'
+    uses: ./.github/workflows/build-test-binaries.yml
+    secrets: inherit
+    permissions:
+      id-token: write
+      contents: read
+    with:
+      test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }}
+      test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}
+      s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
+      terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
+
   EC2LinuxIntegrationTest-0:
     name: 'EC2LinuxTests-0'
-    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload, WarmTestCache ]
+    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload, WarmTestCache, BuildTestBinaries ]
     if: needs.CheckLabel.outputs.should_run == 'true'
     uses:  ./.github/workflows/ec2-integration-test.yml
     permissions:
@@ -223,12 +238,13 @@ jobs:
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
       cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
+      test_binaries_prefix: ${{ needs.BuildTestBinaries.outputs.test_binaries_prefix }}
       is_selinux_test: false
     secrets: inherit
 
   EC2LinuxIntegrationTest-1:
     name: 'EC2LinuxTests-1'
-    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload, WarmTestCache ]
+    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload, WarmTestCache, BuildTestBinaries ]
     if: ${{ needs.CheckLabel.outputs.should_run == 'true' && needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 1 }}
     uses:  ./.github/workflows/ec2-integration-test.yml
     permissions:
@@ -247,12 +263,13 @@ jobs:
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
       cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
+      test_binaries_prefix: ${{ needs.BuildTestBinaries.outputs.test_binaries_prefix }}
       is_selinux_test: false
     secrets: inherit
 
   EC2LinuxIntegrationTest-2:
     name: 'EC2LinuxTests-2'
-    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload, WarmTestCache ]
+    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload, WarmTestCache, BuildTestBinaries ]
     if: ${{ needs.CheckLabel.outputs.should_run == 'true' && needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 2 }}
     uses:  ./.github/workflows/ec2-integration-test.yml
     permissions:
@@ -271,12 +288,13 @@ jobs:
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
       cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
+      test_binaries_prefix: ${{ needs.BuildTestBinaries.outputs.test_binaries_prefix }}
       is_selinux_test: false
     secrets: inherit
 
   EC2LinuxIntegrationTest-3:
     name: 'EC2LinuxTests-3'
-    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload, WarmTestCache ]
+    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload, WarmTestCache, BuildTestBinaries ]
     if: ${{ needs.CheckLabel.outputs.should_run == 'true' && needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 3 }}
     uses:  ./.github/workflows/ec2-integration-test.yml
     permissions:
@@ -295,12 +313,13 @@ jobs:
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
       cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
+      test_binaries_prefix: ${{ needs.BuildTestBinaries.outputs.test_binaries_prefix }}
       is_selinux_test: false
     secrets: inherit
 
   EC2LinuxIntegrationTest-4:
     name: 'EC2LinuxTests-4'
-    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload, WarmTestCache ]
+    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload, WarmTestCache, BuildTestBinaries ]
     if: ${{ needs.CheckLabel.outputs.should_run == 'true' && needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 4 }}
     uses:  ./.github/workflows/ec2-integration-test.yml
     permissions:
@@ -319,12 +338,13 @@ jobs:
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
       cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
+      test_binaries_prefix: ${{ needs.BuildTestBinaries.outputs.test_binaries_prefix }}
       is_selinux_test: false
     secrets: inherit
 
   EC2SELinuxIntegrationTest:
     name: 'EC2SELinuxTests'
-    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload, WarmTestCache ]
+    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload, WarmTestCache, BuildTestBinaries ]
     if: needs.CheckLabel.outputs.should_run == 'true'
     uses:  ./.github/workflows/ec2-integration-test.yml
     permissions:
@@ -343,6 +363,7 @@ jobs:
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
       cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
+      test_binaries_prefix: ${{ needs.BuildTestBinaries.outputs.test_binaries_prefix }}
       is_selinux_test: true
     secrets: inherit
 
@@ -384,7 +405,7 @@ jobs:
   verify-all:
     name: Verify All PR Test Jobs
     needs: [IntegrationTestGate, CheckLabel, BuildAndUpload, OutputEnvVariables, StartLocalStack,
-            GenerateTestMatrix, WarmTestCache, EC2LinuxIntegrationTest-0, EC2LinuxIntegrationTest-1,
+            GenerateTestMatrix, WarmTestCache, BuildTestBinaries, EC2LinuxIntegrationTest-0, EC2LinuxIntegrationTest-1,
             EC2LinuxIntegrationTest-2, EC2LinuxIntegrationTest-3, EC2LinuxIntegrationTest-4,
             EC2SELinuxIntegrationTest, StopLocalStack]
     runs-on: ubuntu-latest
diff --git a/.github/workflows/build-test-binaries.yml b/.github/workflows/build-test-binaries.yml
new file mode 100644
index 0000000000..f7720ef72c
--- /dev/null
+++ b/.github/workflows/build-test-binaries.yml
@@ -0,0 +1,87 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: MIT
+
+name: Build Test Binaries
+
+on:
+  workflow_call:
+    inputs:
+      test_repo_name:
+        required: true
+        type: string
+      test_repo_branch:
+        required: true
+        type: string
+      s3_integration_bucket:
+        required: true
+        type: string
+      terraform_assume_role:
+        required: true
+        type: string
+    outputs:
+      test_binaries_prefix:
+        description: "S3 prefix for pre-compiled test binaries"
+        value: ${{ jobs.BuildTestBinaries.outputs.test_binaries_prefix }}
+
+jobs:
+  BuildTestBinaries:
+    name: 'BuildTestBinaries'
+    runs-on: ubuntu-latest
+    outputs:
+      test_binaries_prefix: ${{ steps.build.outputs.test_binaries_prefix }}
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          repository: ${{ inputs.test_repo_name }}
+          ref: ${{ inputs.test_repo_branch }}
+
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v4
+        with:
+          go-version: ~1.25
+          cache: false
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ inputs.terraform_assume_role }}
+          aws-region: us-west-2
+
+      - name: Build and upload test binaries
+        id: build
+        env:
+          S3_BUCKET: ${{ inputs.s3_integration_bucket }}
+          TEST_BRANCH: ${{ inputs.test_repo_branch }}
+        run: |
+          COMMIT_SHA=$(git rev-parse --short HEAD)
+          PREFIX="integration-test/test-binaries/${COMMIT_SHA}/linux/amd64"
+          echo "test_binaries_prefix=${PREFIX}" >> "$GITHUB_OUTPUT"
+
+          mkdir -p build/bin
+          go mod download
+
+          # Compile each test package under ./test/...
+          FAILED=0
+          for pkg in $(go list ./test/...); do
+            name=$(basename "$pkg")
+            echo "Building ${name}.test from ${pkg}..."
+            if CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go test -c -o "build/bin/${name}.test" "$pkg" 2>&1; then
+              echo "  ✓ ${name}.test"
+            else
+              echo "  ✗ ${name}.test (skipped — may have no tests)"
+              FAILED=$((FAILED + 1))
+            fi
+          done
+
+          # Count results
+          BUILT=$(ls build/bin/*.test 2>/dev/null | wc -l)
+          echo "Built ${BUILT} test binaries (${FAILED} skipped)"
+          ls -lh build/bin/
+
+          # Upload to S3
+          echo "Uploading to s3://${S3_BUCKET}/${PREFIX}/"
+          aws s3 cp build/bin/ "s3://${S3_BUCKET}/${PREFIX}/" --recursive --quiet
+          echo "Upload complete"
diff --git a/.github/workflows/ec2-integration-test.yml b/.github/workflows/ec2-integration-test.yml
index 20eb5cc01e..928d3f2ea8 100644
--- a/.github/workflows/ec2-integration-test.yml
+++ b/.github/workflows/ec2-integration-test.yml
@@ -47,6 +47,9 @@ on:
       cache_key:
         type: string
         required: false
+      test_binaries_prefix:
+        type: string
+        required: false
     secrets:
       AWS_PRIVATE_KEY:
         required: false
@@ -111,6 +114,11 @@ jobs:
               CACHE_KEY_VAR="-var=cache_key=${{ inputs.cache_key }}"
             fi
 
+            TEST_BINARIES_VAR=""
+            if [ -n "${{ inputs.test_binaries_prefix }}" ] && grep -q 'variable "test_binaries_prefix"' variables.tf 2>/dev/null; then
+              TEST_BINARIES_VAR="-var=test_binaries_prefix=${{ inputs.test_binaries_prefix }}"
+            fi
+
             terraform init
             if terraform apply --auto-approve \
               -var="agent_start=${{ matrix.arrays.agentStartCommand }}" \
@@ -137,6 +145,7 @@ jobs:
               -var="test_name=${{ matrix.arrays.os }}" \
               -var="is_onprem=${{ inputs.is_onprem_test }}" \
               $CACHE_KEY_VAR \
+              $TEST_BINARIES_VAR \
               -var="user=${{ matrix.arrays.username }}"; then
               terraform destroy -var="region=${{ inputs.region }}" -var="ami=${{ matrix.arrays.ami }}" -auto-approve
             else
diff --git a/.github/workflows/test-artifacts.yml b/.github/workflows/test-artifacts.yml
index 7f5826a1da..37757e48dc 100644
--- a/.github/workflows/test-artifacts.yml
+++ b/.github/workflows/test-artifacts.yml
@@ -298,6 +298,20 @@ jobs:
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
 
+  BuildTestBinaries:
+    name: 'BuildTestBinaries'
+    needs: [OutputEnvVariables]
+    uses: ./.github/workflows/build-test-binaries.yml
+    secrets: inherit
+    permissions:
+      id-token: write
+      contents: read
+    with:
+      test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }}
+      test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}
+      s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
+      terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
+
   CloudformationTest:
     needs: [OutputEnvVariables, GenerateTestMatrix]
     name: 'CFTest'
@@ -554,7 +568,7 @@ jobs:
             terraform destroy --auto-approve
 
   EC2LinuxIntegrationTest-0:
-    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, WarmTestCache ]
+    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, WarmTestCache, BuildTestBinaries ]
     name: 'EC2Linux-0'
     uses:  ./.github/workflows/ec2-integration-test.yml
     with:
@@ -570,12 +584,13 @@ jobs:
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
       cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
+      test_binaries_prefix: ${{ needs.BuildTestBinaries.outputs.test_binaries_prefix }}
       is_selinux_test: false
       is_onprem_test: false
     secrets: inherit
 
   EC2LinuxIntegrationTest-1:
-    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, WarmTestCache ]
+    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, WarmTestCache, BuildTestBinaries ]
     if: ${{ needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 1 }}
     name: 'EC2Linux-1'
     uses:  ./.github/workflows/ec2-integration-test.yml
@@ -592,12 +607,13 @@ jobs:
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
       cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
+      test_binaries_prefix: ${{ needs.BuildTestBinaries.outputs.test_binaries_prefix }}
       is_selinux_test: false
       is_onprem_test: false
     secrets: inherit
 
   EC2LinuxIntegrationTest-2:
-    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, WarmTestCache ]
+    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, WarmTestCache, BuildTestBinaries ]
     if: ${{ needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 2 }}
     name: 'EC2Linux-2'
     uses:  ./.github/workflows/ec2-integration-test.yml
@@ -614,12 +630,13 @@ jobs:
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
       cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
+      test_binaries_prefix: ${{ needs.BuildTestBinaries.outputs.test_binaries_prefix }}
       is_selinux_test: false
       is_onprem_test: false
     secrets: inherit
 
   EC2LinuxIntegrationTest-3:
-    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, WarmTestCache ]
+    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, WarmTestCache, BuildTestBinaries ]
     if: ${{ needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 3 }}
     name: 'EC2Linux-3'
     uses:  ./.github/workflows/ec2-integration-test.yml
@@ -636,12 +653,13 @@ jobs:
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
       cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
+      test_binaries_prefix: ${{ needs.BuildTestBinaries.outputs.test_binaries_prefix }}
       is_selinux_test: false
       is_onprem_test: false
     secrets: inherit
 
   EC2LinuxIntegrationTest-4:
-    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, WarmTestCache ]
+    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, WarmTestCache, BuildTestBinaries ]
     if: ${{ needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 4 }}
     name: 'EC2Linux-4'
     uses:  ./.github/workflows/ec2-integration-test.yml
@@ -658,12 +676,13 @@ jobs:
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
       cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
+      test_binaries_prefix: ${{ needs.BuildTestBinaries.outputs.test_binaries_prefix }}
       is_selinux_test: false
       is_onprem_test: false
     secrets: inherit
 
   LinuxOnPremIntegrationTest:
-    needs: [GenerateTestMatrix, OutputEnvVariables, WarmTestCache]
+    needs: [GenerateTestMatrix, OutputEnvVariables, WarmTestCache, BuildTestBinaries]
     name: 'OnpremLinux'
     uses: ./.github/workflows/ec2-integration-test.yml
     with:
@@ -679,6 +698,7 @@ jobs:
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
       cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
+      test_binaries_prefix: ${{ needs.BuildTestBinaries.outputs.test_binaries_prefix }}
       is_selinux_test: false
       is_onprem_test: true
     secrets: inherit
@@ -725,7 +745,7 @@ jobs:
       KEY_NAME: ${{ secrets.KEY_NAME }}
 
   EC2SELinuxIntegrationTest:
-    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, WarmTestCache ]
+    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, WarmTestCache, BuildTestBinaries ]
     name: 'EC2SELinux'
     uses:  ./.github/workflows/ec2-integration-test.yml
     with:
@@ -741,6 +761,7 @@ jobs:
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
       cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
+      test_binaries_prefix: ${{ needs.BuildTestBinaries.outputs.test_binaries_prefix }}
       is_selinux_test: true
 
   EC2WinIntegrationTest:

From c4baf2655e2dac64d6c46eda422eb8d51aa63d49 Mon Sep 17 00:00:00 2001
From: Marcus Mann <mpmann@amazon.com>
Date: Tue, 17 Feb 2026 21:00:59 +0000
Subject: [PATCH 06/19] fix(ci): Build test binaries for both amd64 and arm64
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The test matrix includes arm64 instances. Build binaries for both
architectures and output the prefix without arch so terraform can
append /linux/${arc} to select the correct binary.

🤖 Assisted by AI
---
 .github/workflows/build-test-binaries.yml | 40 ++++++++++-------------
 1 file changed, 18 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/build-test-binaries.yml b/.github/workflows/build-test-binaries.yml
index f7720ef72c..ff3f83fccd 100644
--- a/.github/workflows/build-test-binaries.yml
+++ b/.github/workflows/build-test-binaries.yml
@@ -57,31 +57,27 @@ jobs:
           TEST_BRANCH: ${{ inputs.test_repo_branch }}
         run: |
           COMMIT_SHA=$(git rev-parse --short HEAD)
-          PREFIX="integration-test/test-binaries/${COMMIT_SHA}/linux/amd64"
-          echo "test_binaries_prefix=${PREFIX}" >> "$GITHUB_OUTPUT"
+          echo "test_binaries_prefix=integration-test/test-binaries/${COMMIT_SHA}" >> "$GITHUB_OUTPUT"
 
-          mkdir -p build/bin
           go mod download
 
-          # Compile each test package under ./test/...
-          FAILED=0
-          for pkg in $(go list ./test/...); do
-            name=$(basename "$pkg")
-            echo "Building ${name}.test from ${pkg}..."
-            if CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go test -c -o "build/bin/${name}.test" "$pkg" 2>&1; then
-              echo "  ✓ ${name}.test"
-            else
-              echo "  ✗ ${name}.test (skipped — may have no tests)"
-              FAILED=$((FAILED + 1))
-            fi
-          done
+          for GOARCH in amd64 arm64; do
+            PREFIX="integration-test/test-binaries/${COMMIT_SHA}/linux/${GOARCH}"
+            mkdir -p "build/bin/${GOARCH}"
+
+            echo "=== Building for linux/${GOARCH} ==="
+            for pkg in $(go list ./test/...); do
+              name=$(basename "$pkg")
+              if CGO_ENABLED=0 GOOS=linux GOARCH=$GOARCH go test -c -o "build/bin/${GOARCH}/${name}.test" "$pkg" 2>&1; then
+                echo "  ✓ ${name}.test"
+              else
+                echo "  ✗ ${name}.test (skipped)"
+              fi
+            done
 
-          # Count results
-          BUILT=$(ls build/bin/*.test 2>/dev/null | wc -l)
-          echo "Built ${BUILT} test binaries (${FAILED} skipped)"
-          ls -lh build/bin/
+            BUILT=$(ls "build/bin/${GOARCH}/"*.test 2>/dev/null | wc -l)
+            echo "Built ${BUILT} binaries for linux/${GOARCH}"
 
-          # Upload to S3
-          echo "Uploading to s3://${S3_BUCKET}/${PREFIX}/"
-          aws s3 cp build/bin/ "s3://${S3_BUCKET}/${PREFIX}/" --recursive --quiet
+            aws s3 cp "build/bin/${GOARCH}/" "s3://${S3_BUCKET}/${PREFIX}/" --recursive --quiet
+          done
           echo "Upload complete"

From 49a8e7d0fd6d4f0036c8fcc9a3582a0d9b4bfe3b Mon Sep 17 00:00:00 2001
From: Marcus Mann <mpmann@amazon.com>
Date: Thu, 19 Feb 2026 16:39:20 -0500
Subject: [PATCH 07/19] refactor(ci): Remove Go cache warming, keep only
 pre-compiled binaries

The WarmTestCache job added complexity without significant benefit compared
to pre-compiled binaries. Removing it simplifies the workflow.

Removed:
- .github/workflows/warm-test-cache.yml
- cache_key input from ec2-integration-test.yml
- WarmTestCache job references from PR-test.yml and test-artifacts.yml

The BuildTestBinaries job remains - that's where the real savings come from.
---
 .github/workflows/PR-test.yml              | 35 ++-------
 .github/workflows/ec2-integration-test.yml |  9 ---
 .github/workflows/test-artifacts.yml       | 35 ++-------
 .github/workflows/warm-test-cache.yml      | 84 ----------------------
 4 files changed, 14 insertions(+), 149 deletions(-)
 delete mode 100644 .github/workflows/warm-test-cache.yml

diff --git a/.github/workflows/PR-test.yml b/.github/workflows/PR-test.yml
index 712cdbc5a5..76e66db3a1 100644
--- a/.github/workflows/PR-test.yml
+++ b/.github/workflows/PR-test.yml
@@ -187,21 +187,6 @@ jobs:
           echo "ec2_linux_matrix pages: ${{ steps.paginate-matrix.outputs.ec2_linux_matrix_page_count }}"
           echo "ec2_selinux_matrix: ${{ steps.set-matrix.outputs.ec2_selinux_matrix }}"
 
-  WarmTestCache:
-    name: 'WarmTestCache'
-    needs: [CheckLabel, OutputEnvVariables]
-    if: needs.CheckLabel.outputs.should_run == 'true'
-    uses: ./.github/workflows/warm-test-cache.yml
-    secrets: inherit
-    permissions:
-      id-token: write
-      contents: read
-    with:
-      test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }}
-      test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}
-      s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
-      terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
-
   BuildTestBinaries:
     name: 'BuildTestBinaries'
     needs: [CheckLabel, OutputEnvVariables]
@@ -219,7 +204,7 @@ jobs:
 
   EC2LinuxIntegrationTest-0:
     name: 'EC2LinuxTests-0'
-    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload, WarmTestCache, BuildTestBinaries ]
+    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload, BuildTestBinaries ]
     if: needs.CheckLabel.outputs.should_run == 'true'
     uses:  ./.github/workflows/ec2-integration-test.yml
     permissions:
@@ -237,14 +222,13 @@ jobs:
       region: us-west-2
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
-      cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
       test_binaries_prefix: ${{ needs.BuildTestBinaries.outputs.test_binaries_prefix }}
       is_selinux_test: false
     secrets: inherit
 
   EC2LinuxIntegrationTest-1:
     name: 'EC2LinuxTests-1'
-    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload, WarmTestCache, BuildTestBinaries ]
+    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload, BuildTestBinaries ]
     if: ${{ needs.CheckLabel.outputs.should_run == 'true' && needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 1 }}
     uses:  ./.github/workflows/ec2-integration-test.yml
     permissions:
@@ -262,14 +246,13 @@ jobs:
       region: us-west-2
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
-      cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
       test_binaries_prefix: ${{ needs.BuildTestBinaries.outputs.test_binaries_prefix }}
       is_selinux_test: false
     secrets: inherit
 
   EC2LinuxIntegrationTest-2:
     name: 'EC2LinuxTests-2'
-    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload, WarmTestCache, BuildTestBinaries ]
+    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload, BuildTestBinaries ]
     if: ${{ needs.CheckLabel.outputs.should_run == 'true' && needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 2 }}
     uses:  ./.github/workflows/ec2-integration-test.yml
     permissions:
@@ -287,14 +270,13 @@ jobs:
       region: us-west-2
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
-      cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
       test_binaries_prefix: ${{ needs.BuildTestBinaries.outputs.test_binaries_prefix }}
       is_selinux_test: false
     secrets: inherit
 
   EC2LinuxIntegrationTest-3:
     name: 'EC2LinuxTests-3'
-    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload, WarmTestCache, BuildTestBinaries ]
+    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload, BuildTestBinaries ]
     if: ${{ needs.CheckLabel.outputs.should_run == 'true' && needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 3 }}
     uses:  ./.github/workflows/ec2-integration-test.yml
     permissions:
@@ -312,14 +294,13 @@ jobs:
       region: us-west-2
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
-      cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
       test_binaries_prefix: ${{ needs.BuildTestBinaries.outputs.test_binaries_prefix }}
       is_selinux_test: false
     secrets: inherit
 
   EC2LinuxIntegrationTest-4:
     name: 'EC2LinuxTests-4'
-    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload, WarmTestCache, BuildTestBinaries ]
+    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload, BuildTestBinaries ]
     if: ${{ needs.CheckLabel.outputs.should_run == 'true' && needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 4 }}
     uses:  ./.github/workflows/ec2-integration-test.yml
     permissions:
@@ -337,14 +318,13 @@ jobs:
       region: us-west-2
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
-      cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
       test_binaries_prefix: ${{ needs.BuildTestBinaries.outputs.test_binaries_prefix }}
       is_selinux_test: false
     secrets: inherit
 
   EC2SELinuxIntegrationTest:
     name: 'EC2SELinuxTests'
-    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload, WarmTestCache, BuildTestBinaries ]
+    needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload, BuildTestBinaries ]
     if: needs.CheckLabel.outputs.should_run == 'true'
     uses:  ./.github/workflows/ec2-integration-test.yml
     permissions:
@@ -362,7 +342,6 @@ jobs:
       region: us-west-2
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
-      cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
       test_binaries_prefix: ${{ needs.BuildTestBinaries.outputs.test_binaries_prefix }}
       is_selinux_test: true
     secrets: inherit
@@ -405,7 +384,7 @@ jobs:
   verify-all:
     name: Verify All PR Test Jobs
     needs: [IntegrationTestGate, CheckLabel, BuildAndUpload, OutputEnvVariables, StartLocalStack,
-            GenerateTestMatrix, WarmTestCache, BuildTestBinaries, EC2LinuxIntegrationTest-0, EC2LinuxIntegrationTest-1,
+            GenerateTestMatrix, BuildTestBinaries, EC2LinuxIntegrationTest-0, EC2LinuxIntegrationTest-1,
             EC2LinuxIntegrationTest-2, EC2LinuxIntegrationTest-3, EC2LinuxIntegrationTest-4,
             EC2SELinuxIntegrationTest, StopLocalStack]
     runs-on: ubuntu-latest
diff --git a/.github/workflows/ec2-integration-test.yml b/.github/workflows/ec2-integration-test.yml
index 928d3f2ea8..1a9138f596 100644
--- a/.github/workflows/ec2-integration-test.yml
+++ b/.github/workflows/ec2-integration-test.yml
@@ -44,9 +44,6 @@ on:
         type: boolean
       s3_integration_bucket:
         type: string
-      cache_key:
-        type: string
-        required: false
       test_binaries_prefix:
         type: string
         required: false
@@ -109,11 +106,6 @@ jobs:
               cd ${{inputs.test_dir}}
             fi
 
-            CACHE_KEY_VAR=""
-            if [ -n "${{ inputs.cache_key }}" ] && grep -q 'variable "cache_key"' variables.tf 2>/dev/null; then
-              CACHE_KEY_VAR="-var=cache_key=${{ inputs.cache_key }}"
-            fi
-
             TEST_BINARIES_VAR=""
             if [ -n "${{ inputs.test_binaries_prefix }}" ] && grep -q 'variable "test_binaries_prefix"' variables.tf 2>/dev/null; then
               TEST_BINARIES_VAR="-var=test_binaries_prefix=${{ inputs.test_binaries_prefix }}"
@@ -144,7 +136,6 @@ jobs:
               -var="test_dir=${{ matrix.arrays.test_dir }}" \
               -var="test_name=${{ matrix.arrays.os }}" \
               -var="is_onprem=${{ inputs.is_onprem_test }}" \
-              $CACHE_KEY_VAR \
               $TEST_BINARIES_VAR \
               -var="user=${{ matrix.arrays.username }}"; then
               terraform destroy -var="region=${{ inputs.region }}" -var="ami=${{ matrix.arrays.ami }}" -auto-approve
diff --git a/.github/workflows/test-artifacts.yml b/.github/workflows/test-artifacts.yml
index 37757e48dc..9b1d260b32 100644
--- a/.github/workflows/test-artifacts.yml
+++ b/.github/workflows/test-artifacts.yml
@@ -284,20 +284,6 @@ jobs:
           echo "ec2_linux_itar_matrix: ${{ steps.set-matrix.outputs.ec2_linux_itar_matrix }}"
           echo "ec2_linux_china_matrix: ${{ steps.set-matrix.outputs.ec2_linux_china_matrix }}"
 
-  WarmTestCache:
-    name: 'WarmTestCache'
-    needs: [OutputEnvVariables]
-    uses: ./.github/workflows/warm-test-cache.yml
-    secrets: inherit
-    permissions:
-      id-token: write
-      contents: read
-    with:
-      test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }}
-      test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}
-      s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
-      terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
-
   BuildTestBinaries:
     name: 'BuildTestBinaries'
     needs: [OutputEnvVariables]
@@ -568,7 +554,7 @@ jobs:
             terraform destroy --auto-approve
 
   EC2LinuxIntegrationTest-0:
-    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, WarmTestCache, BuildTestBinaries ]
+    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, BuildTestBinaries ]
     name: 'EC2Linux-0'
     uses:  ./.github/workflows/ec2-integration-test.yml
     with:
@@ -583,14 +569,13 @@ jobs:
       region: us-west-2
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
-      cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
       test_binaries_prefix: ${{ needs.BuildTestBinaries.outputs.test_binaries_prefix }}
       is_selinux_test: false
       is_onprem_test: false
     secrets: inherit
 
   EC2LinuxIntegrationTest-1:
-    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, WarmTestCache, BuildTestBinaries ]
+    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, BuildTestBinaries ]
     if: ${{ needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 1 }}
     name: 'EC2Linux-1'
     uses:  ./.github/workflows/ec2-integration-test.yml
@@ -606,14 +591,13 @@ jobs:
       region: us-west-2
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
-      cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
       test_binaries_prefix: ${{ needs.BuildTestBinaries.outputs.test_binaries_prefix }}
       is_selinux_test: false
       is_onprem_test: false
     secrets: inherit
 
   EC2LinuxIntegrationTest-2:
-    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, WarmTestCache, BuildTestBinaries ]
+    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, BuildTestBinaries ]
     if: ${{ needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 2 }}
     name: 'EC2Linux-2'
     uses:  ./.github/workflows/ec2-integration-test.yml
@@ -629,14 +613,13 @@ jobs:
       region: us-west-2
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
-      cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
       test_binaries_prefix: ${{ needs.BuildTestBinaries.outputs.test_binaries_prefix }}
       is_selinux_test: false
       is_onprem_test: false
     secrets: inherit
 
   EC2LinuxIntegrationTest-3:
-    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, WarmTestCache, BuildTestBinaries ]
+    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, BuildTestBinaries ]
     if: ${{ needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 3 }}
     name: 'EC2Linux-3'
     uses:  ./.github/workflows/ec2-integration-test.yml
@@ -652,14 +635,13 @@ jobs:
       region: us-west-2
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
-      cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
       test_binaries_prefix: ${{ needs.BuildTestBinaries.outputs.test_binaries_prefix }}
       is_selinux_test: false
       is_onprem_test: false
     secrets: inherit
 
   EC2LinuxIntegrationTest-4:
-    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, WarmTestCache, BuildTestBinaries ]
+    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, BuildTestBinaries ]
     if: ${{ needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 4 }}
     name: 'EC2Linux-4'
     uses:  ./.github/workflows/ec2-integration-test.yml
@@ -675,14 +657,13 @@ jobs:
       region: us-west-2
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
-      cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
       test_binaries_prefix: ${{ needs.BuildTestBinaries.outputs.test_binaries_prefix }}
       is_selinux_test: false
       is_onprem_test: false
     secrets: inherit
 
   LinuxOnPremIntegrationTest:
-    needs: [GenerateTestMatrix, OutputEnvVariables, WarmTestCache, BuildTestBinaries]
+    needs: [GenerateTestMatrix, OutputEnvVariables, BuildTestBinaries]
     name: 'OnpremLinux'
     uses: ./.github/workflows/ec2-integration-test.yml
     with:
@@ -697,7 +678,6 @@ jobs:
       region: us-west-2
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
-      cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
       test_binaries_prefix: ${{ needs.BuildTestBinaries.outputs.test_binaries_prefix }}
       is_selinux_test: false
       is_onprem_test: true
@@ -745,7 +725,7 @@ jobs:
       KEY_NAME: ${{ secrets.KEY_NAME }}
 
   EC2SELinuxIntegrationTest:
-    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, WarmTestCache, BuildTestBinaries ]
+    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, BuildTestBinaries ]
     name: 'EC2SELinux'
     uses:  ./.github/workflows/ec2-integration-test.yml
     with:
@@ -760,7 +740,6 @@ jobs:
       region: us-west-2
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
-      cache_key: ${{ needs.WarmTestCache.outputs.cache_key }}
       test_binaries_prefix: ${{ needs.BuildTestBinaries.outputs.test_binaries_prefix }}
       is_selinux_test: true
 
diff --git a/.github/workflows/warm-test-cache.yml b/.github/workflows/warm-test-cache.yml
deleted file mode 100644
index 0d61e0d1e5..0000000000
--- a/.github/workflows/warm-test-cache.yml
+++ /dev/null
@@ -1,84 +0,0 @@
-# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
-# SPDX-License-Identifier: MIT
-
-name: Warm Go Test Cache
-
-on:
-  workflow_call:
-    inputs:
-      test_repo_name:
-        required: true
-        type: string
-      test_repo_branch:
-        required: true
-        type: string
-      s3_integration_bucket:
-        required: true
-        type: string
-      terraform_assume_role:
-        required: true
-        type: string
-    outputs:
-      cache_key:
-        description: "Cache key for restoring Go build/module caches"
-        value: ${{ jobs.WarmTestCache.outputs.cache_key }}
-
-jobs:
-  WarmTestCache:
-    name: 'WarmTestCache'
-    runs-on: ubuntu-latest
-    outputs:
-      cache_key: ${{ steps.warm.outputs.cache_key }}
-    permissions:
-      id-token: write
-      contents: read
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: ${{ inputs.test_repo_name }}
-          ref: ${{ inputs.test_repo_branch }}
-
-      - name: Set up Go 1.x
-        uses: actions/setup-go@v4
-        with:
-          go-version: ~1.25
-          cache: false
-
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          role-to-assume: ${{ inputs.terraform_assume_role }}
-          aws-region: us-west-2
-
-      - name: Warm cache and upload to S3
-        id: warm
-        env:
-          S3_BUCKET: ${{ inputs.s3_integration_bucket }}
-          TEST_BRANCH: ${{ inputs.test_repo_branch }}
-        run: |
-          GO_VERSION=$(go env GOVERSION)
-          CACHE_KEY="${TEST_BRANCH}-${GO_VERSION}-linux-amd64"
-          echo "cache_key=${CACHE_KEY}" >> "$GITHUB_OUTPUT"
-          echo "Cache key: ${CACHE_KEY}"
-
-          # Download modules and compile all test packages (without running them)
-          go mod download
-          CGO_ENABLED=0 go test -run='^$' ./test/... 2>/dev/null || true
-
-          # Upload caches to S3
-          GOCACHE=$(go env GOCACHE)
-          GOMODCACHE=$(go env GOMODCACHE)
-          CACHE_PREFIX="s3://${S3_BUCKET}/integration-test/cache/${CACHE_KEY}"
-
-          echo "Compressing build cache from ${GOCACHE}..."
-          tar czf /tmp/gocache.tar.gz -C "$GOCACHE" .
-          echo "Compressing module cache from ${GOMODCACHE}..."
-          tar czf /tmp/gomodcache.tar.gz -C "$GOMODCACHE" .
-
-          echo "Uploading to ${CACHE_PREFIX}..."
-          aws s3 cp /tmp/gocache.tar.gz "${CACHE_PREFIX}/gocache.tar.gz" --quiet
-          aws s3 cp /tmp/gomodcache.tar.gz "${CACHE_PREFIX}/gomodcache.tar.gz" --quiet
-
-          GOCACHE_SIZE=$(du -sh /tmp/gocache.tar.gz | cut -f1)
-          GOMODCACHE_SIZE=$(du -sh /tmp/gomodcache.tar.gz | cut -f1)
-          echo "Build cache: ${GOCACHE_SIZE}, Module cache: ${GOMODCACHE_SIZE}"

From aeda280ca4bd4280b82d5f44533a987c7a09f50c Mon Sep 17 00:00:00 2001
From: Marcus Mann <mpmann@amazon.com>
Date: Fri, 20 Feb 2026 10:26:31 -0500
Subject: [PATCH 08/19] feat: Upload test binaries to ITAR and CN S3 buckets

- Add ITAR/CN bucket inputs to build-test-binaries.yml
- Upload test binaries to all three regions (commercial, ITAR, CN)
- Pass test_binaries_prefix to ITAR and CN integration test jobs
- ITAR/CN will now use pre-compiled binaries instead of go test
---
 .github/workflows/build-test-binaries.yml | 54 ++++++++++++++++++++++-
 .github/workflows/test-artifacts.yml      | 10 ++++-
 2 files changed, 61 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build-test-binaries.yml b/.github/workflows/build-test-binaries.yml
index ff3f83fccd..89339ddd20 100644
--- a/.github/workflows/build-test-binaries.yml
+++ b/.github/workflows/build-test-binaries.yml
@@ -18,6 +18,18 @@ on:
       terraform_assume_role:
         required: true
         type: string
+      s3_integration_bucket_itar:
+        required: false
+        type: string
+      terraform_assume_role_itar:
+        required: false
+        type: string
+      s3_integration_bucket_cn:
+        required: false
+        type: string
+      terraform_assume_role_cn:
+        required: false
+        type: string
     outputs:
       test_binaries_prefix:
         description: "S3 prefix for pre-compiled test binaries"
@@ -80,4 +92,44 @@ jobs:
 
             aws s3 cp "build/bin/${GOARCH}/" "s3://${S3_BUCKET}/${PREFIX}/" --recursive --quiet
           done
-          echo "Upload complete"
+          echo "Upload complete to commercial bucket"
+
+      - name: Upload to ITAR bucket
+        if: ${{ inputs.s3_integration_bucket_itar != '' && inputs.terraform_assume_role_itar != '' }}
+        env:
+          S3_BUCKET_ITAR: ${{ inputs.s3_integration_bucket_itar }}
+          ASSUME_ROLE_ITAR: ${{ inputs.terraform_assume_role_itar }}
+        run: |
+          COMMIT_SHA=$(git rev-parse --short HEAD)
+          
+          # Get ITAR credentials
+          CREDS=$(aws sts assume-role --role-arn "$ASSUME_ROLE_ITAR" --role-session-name "test-binaries-itar" --query 'Credentials' --output json)
+          export AWS_ACCESS_KEY_ID=$(echo $CREDS | jq -r '.AccessKeyId')
+          export AWS_SECRET_ACCESS_KEY=$(echo $CREDS | jq -r '.SecretAccessKey')
+          export AWS_SESSION_TOKEN=$(echo $CREDS | jq -r '.SessionToken')
+          
+          for GOARCH in amd64 arm64; do
+            PREFIX="integration-test/test-binaries/${COMMIT_SHA}/linux/${GOARCH}"
+            aws s3 cp "build/bin/${GOARCH}/" "s3://${S3_BUCKET_ITAR}/${PREFIX}/" --recursive --quiet --region us-gov-east-1
+          done
+          echo "Upload complete to ITAR bucket"
+
+      - name: Upload to CN bucket
+        if: ${{ inputs.s3_integration_bucket_cn != '' && inputs.terraform_assume_role_cn != '' }}
+        env:
+          S3_BUCKET_CN: ${{ inputs.s3_integration_bucket_cn }}
+          ASSUME_ROLE_CN: ${{ inputs.terraform_assume_role_cn }}
+        run: |
+          COMMIT_SHA=$(git rev-parse --short HEAD)
+          
+          # Get CN credentials
+          CREDS=$(aws sts assume-role --role-arn "$ASSUME_ROLE_CN" --role-session-name "test-binaries-cn" --query 'Credentials' --output json)
+          export AWS_ACCESS_KEY_ID=$(echo $CREDS | jq -r '.AccessKeyId')
+          export AWS_SECRET_ACCESS_KEY=$(echo $CREDS | jq -r '.SecretAccessKey')
+          export AWS_SESSION_TOKEN=$(echo $CREDS | jq -r '.SessionToken')
+          
+          for GOARCH in amd64 arm64; do
+            PREFIX="integration-test/test-binaries/${COMMIT_SHA}/linux/${GOARCH}"
+            aws s3 cp "build/bin/${GOARCH}/" "s3://${S3_BUCKET_CN}/${PREFIX}/" --recursive --quiet --region cn-north-1
+          done
+          echo "Upload complete to CN bucket"
diff --git a/.github/workflows/test-artifacts.yml b/.github/workflows/test-artifacts.yml
index 9b1d260b32..64a4d6f98d 100644
--- a/.github/workflows/test-artifacts.yml
+++ b/.github/workflows/test-artifacts.yml
@@ -297,6 +297,10 @@ jobs:
       test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }}
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
+      s3_integration_bucket_itar: ${{ vars.S3_INTEGRATION_BUCKET_ITAR }}
+      terraform_assume_role_itar: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_ITAR }}
+      s3_integration_bucket_cn: ${{ vars.S3_INTEGRATION_BUCKET_CN }}
+      terraform_assume_role_cn: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_CN }}
 
   CloudformationTest:
     needs: [OutputEnvVariables, GenerateTestMatrix]
@@ -685,7 +689,7 @@ jobs:
 
 
   EC2LinuxIntegrationTestITAR:
-    needs: [ StartLocalStackITAR, GenerateTestMatrix, OutputEnvVariables ]
+    needs: [ StartLocalStackITAR, GenerateTestMatrix, OutputEnvVariables, BuildTestBinaries ]
     name: 'EC2LinuxITAR'
     uses: ./.github/workflows/ec2-integration-test.yml
     with:
@@ -700,12 +704,13 @@ jobs:
       region: us-gov-east-1
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_ITAR }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET_ITAR }}
+      test_binaries_prefix: ${{ needs.BuildTestBinaries.outputs.test_binaries_prefix }}
     secrets:
       AWS_PRIVATE_KEY: ${{ secrets.AWS_PRIVATE_KEY_ITAR }}
       KEY_NAME: ${{ secrets.KEY_NAME }}
 
   EC2LinuxIntegrationTestCN:
-    needs: [ StartLocalStackCN, GenerateTestMatrix, OutputEnvVariables ]
+    needs: [ StartLocalStackCN, GenerateTestMatrix, OutputEnvVariables, BuildTestBinaries ]
     name: 'EC2LinuxCN'
     uses: ./.github/workflows/ec2-integration-test.yml
     with:
@@ -720,6 +725,7 @@ jobs:
       region: cn-north-1
       terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_CN }}
       s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET_CN }}
+      test_binaries_prefix: ${{ needs.BuildTestBinaries.outputs.test_binaries_prefix }}
     secrets:
       AWS_PRIVATE_KEY: ${{ secrets.AWS_PRIVATE_KEY_CN }}
       KEY_NAME: ${{ secrets.KEY_NAME }}

From 09477042e1b4b82408777c5104598c9410455620 Mon Sep 17 00:00:00 2001
From: Marcus Mann <mpmann@amazon.com>
Date: Fri, 20 Feb 2026 11:03:37 -0500
Subject: [PATCH 09/19] fix: Use separate jobs with OIDC auth for ITAR/CN
 uploads

Cross-partition assume-role doesn't work. Each partition needs its own
OIDC authentication via configure-aws-credentials.

- Split ITAR/CN uploads into separate jobs
- Use GitHub artifacts to pass binaries between jobs
- Add Complete job to ensure all uploads finish before tests start
---
 .github/workflows/build-test-binaries.yml | 101 ++++++++++++++++------
 1 file changed, 73 insertions(+), 28 deletions(-)

diff --git a/.github/workflows/build-test-binaries.yml b/.github/workflows/build-test-binaries.yml
index 89339ddd20..63bb862ba2 100644
--- a/.github/workflows/build-test-binaries.yml
+++ b/.github/workflows/build-test-binaries.yml
@@ -33,7 +33,7 @@ on:
     outputs:
       test_binaries_prefix:
         description: "S3 prefix for pre-compiled test binaries"
-        value: ${{ jobs.BuildTestBinaries.outputs.test_binaries_prefix }}
+        value: ${{ jobs.Complete.outputs.test_binaries_prefix }}
 
 jobs:
   BuildTestBinaries:
@@ -66,7 +66,6 @@ jobs:
         id: build
         env:
           S3_BUCKET: ${{ inputs.s3_integration_bucket }}
-          TEST_BRANCH: ${{ inputs.test_repo_branch }}
         run: |
           COMMIT_SHA=$(git rev-parse --short HEAD)
           echo "test_binaries_prefix=integration-test/test-binaries/${COMMIT_SHA}" >> "$GITHUB_OUTPUT"
@@ -94,42 +93,88 @@ jobs:
           done
           echo "Upload complete to commercial bucket"
 
+      - name: Upload binaries artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-binaries
+          path: build/bin/
+          retention-days: 1
+
+  UploadTestBinariesITAR:
+    name: 'UploadTestBinariesITAR'
+    needs: [BuildTestBinaries]
+    if: ${{ inputs.s3_integration_bucket_itar != '' && inputs.terraform_assume_role_itar != '' }}
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Download binaries artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: test-binaries
+          path: build/bin/
+
+      - name: Configure AWS Credentials for ITAR
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ inputs.terraform_assume_role_itar }}
+          aws-region: us-gov-east-1
+
       - name: Upload to ITAR bucket
-        if: ${{ inputs.s3_integration_bucket_itar != '' && inputs.terraform_assume_role_itar != '' }}
         env:
-          S3_BUCKET_ITAR: ${{ inputs.s3_integration_bucket_itar }}
-          ASSUME_ROLE_ITAR: ${{ inputs.terraform_assume_role_itar }}
+          S3_BUCKET: ${{ inputs.s3_integration_bucket_itar }}
+          PREFIX: ${{ needs.BuildTestBinaries.outputs.test_binaries_prefix }}
         run: |
-          COMMIT_SHA=$(git rev-parse --short HEAD)
-          
-          # Get ITAR credentials
-          CREDS=$(aws sts assume-role --role-arn "$ASSUME_ROLE_ITAR" --role-session-name "test-binaries-itar" --query 'Credentials' --output json)
-          export AWS_ACCESS_KEY_ID=$(echo $CREDS | jq -r '.AccessKeyId')
-          export AWS_SECRET_ACCESS_KEY=$(echo $CREDS | jq -r '.SecretAccessKey')
-          export AWS_SESSION_TOKEN=$(echo $CREDS | jq -r '.SessionToken')
-          
           for GOARCH in amd64 arm64; do
-            PREFIX="integration-test/test-binaries/${COMMIT_SHA}/linux/${GOARCH}"
-            aws s3 cp "build/bin/${GOARCH}/" "s3://${S3_BUCKET_ITAR}/${PREFIX}/" --recursive --quiet --region us-gov-east-1
+            aws s3 cp "build/bin/${GOARCH}/" "s3://${S3_BUCKET}/${PREFIX}/linux/${GOARCH}/" --recursive --quiet
           done
           echo "Upload complete to ITAR bucket"
 
+  UploadTestBinariesCN:
+    name: 'UploadTestBinariesCN'
+    needs: [BuildTestBinaries]
+    if: ${{ inputs.s3_integration_bucket_cn != '' && inputs.terraform_assume_role_cn != '' }}
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Download binaries artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: test-binaries
+          path: build/bin/
+
+      - name: Configure AWS Credentials for CN
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ inputs.terraform_assume_role_cn }}
+          aws-region: cn-north-1
+
       - name: Upload to CN bucket
-        if: ${{ inputs.s3_integration_bucket_cn != '' && inputs.terraform_assume_role_cn != '' }}
         env:
-          S3_BUCKET_CN: ${{ inputs.s3_integration_bucket_cn }}
-          ASSUME_ROLE_CN: ${{ inputs.terraform_assume_role_cn }}
+          S3_BUCKET: ${{ inputs.s3_integration_bucket_cn }}
+          PREFIX: ${{ needs.BuildTestBinaries.outputs.test_binaries_prefix }}
         run: |
-          COMMIT_SHA=$(git rev-parse --short HEAD)
-          
-          # Get CN credentials
-          CREDS=$(aws sts assume-role --role-arn "$ASSUME_ROLE_CN" --role-session-name "test-binaries-cn" --query 'Credentials' --output json)
-          export AWS_ACCESS_KEY_ID=$(echo $CREDS | jq -r '.AccessKeyId')
-          export AWS_SECRET_ACCESS_KEY=$(echo $CREDS | jq -r '.SecretAccessKey')
-          export AWS_SESSION_TOKEN=$(echo $CREDS | jq -r '.SessionToken')
-          
           for GOARCH in amd64 arm64; do
-            PREFIX="integration-test/test-binaries/${COMMIT_SHA}/linux/${GOARCH}"
-            aws s3 cp "build/bin/${GOARCH}/" "s3://${S3_BUCKET_CN}/${PREFIX}/" --recursive --quiet --region cn-north-1
+            aws s3 cp "build/bin/${GOARCH}/" "s3://${S3_BUCKET}/${PREFIX}/linux/${GOARCH}/" --recursive --quiet
           done
           echo "Upload complete to CN bucket"
+
+  Complete:
+    name: 'Complete'
+    needs: [BuildTestBinaries, UploadTestBinariesITAR, UploadTestBinariesCN]
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    outputs:
+      test_binaries_prefix: ${{ needs.BuildTestBinaries.outputs.test_binaries_prefix }}
+    steps:
+      - name: Check results
+        run: |
+          echo "BuildTestBinaries: ${{ needs.BuildTestBinaries.result }}"
+          echo "UploadTestBinariesITAR: ${{ needs.UploadTestBinariesITAR.result }}"
+          echo "UploadTestBinariesCN: ${{ needs.UploadTestBinariesCN.result }}"
+          if [[ "${{ needs.BuildTestBinaries.result }}" != "success" ]]; then
+            exit 1
+          fi

From 1cc298c2216b3f9617e942c34a867eac96c3d15c Mon Sep 17 00:00:00 2001
From: Marcus Mann <mpmann@amazon.com>
Date: Fri, 20 Feb 2026 11:54:43 -0500
Subject: [PATCH 10/19] feat: Pass test_func to terraform for per-function test
 execution

---
 .github/workflows/ec2-integration-test.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ec2-integration-test.yml b/.github/workflows/ec2-integration-test.yml
index 1a9138f596..985fc728fa 100644
--- a/.github/workflows/ec2-integration-test.yml
+++ b/.github/workflows/ec2-integration-test.yml
@@ -135,6 +135,7 @@ jobs:
               -var="ssh_key_value=${{env.PRIVATE_KEY}}" \
               -var="test_dir=${{ matrix.arrays.test_dir }}" \
               -var="test_name=${{ matrix.arrays.os }}" \
+              -var="test_func=${{ matrix.arrays.testFunc }}" \
               -var="is_onprem=${{ inputs.is_onprem_test }}" \
               $TEST_BINARIES_VAR \
               -var="user=${{ matrix.arrays.username }}"; then

From 307372c7d41143afc4f96b6484d563353c725229 Mon Sep 17 00:00:00 2001
From: Marcus Mann <mpmann@amazon.com>
Date: Fri, 20 Feb 2026 11:57:58 -0500
Subject: [PATCH 11/19] feat: Parallelize test binary and Mac package builds

- Split test binary builds into parallel amd64/arm64 jobs
- Split MakeMacPkg into parallel matrix jobs for each architecture
- Add single-arch Makefile targets for darwin builds

Expected improvements:
- Test binary build: ~50% faster (parallel arch builds)
- MakeMacPkg: ~50% faster (parallel arch builds)
---
 .github/workflows/build-test-binaries.yml | 77 ++++++++++++++---------
 .github/workflows/test-build-packages.yml | 31 +++++----
 Makefile                                  | 70 +++++++++++++++++++++
 3 files changed, 135 insertions(+), 43 deletions(-)

diff --git a/.github/workflows/build-test-binaries.yml b/.github/workflows/build-test-binaries.yml
index 63bb862ba2..9f154e4e50 100644
--- a/.github/workflows/build-test-binaries.yml
+++ b/.github/workflows/build-test-binaries.yml
@@ -37,10 +37,13 @@ on:
 
 jobs:
   BuildTestBinaries:
-    name: 'BuildTestBinaries'
+    name: 'BuildTestBinaries-${{ matrix.arch }}'
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        arch: [amd64, arm64]
     outputs:
-      test_binaries_prefix: ${{ steps.build.outputs.test_binaries_prefix }}
+      test_binaries_prefix: ${{ steps.set-prefix.outputs.test_binaries_prefix }}
     permissions:
       id-token: write
       contents: read
@@ -50,6 +53,12 @@ jobs:
           repository: ${{ inputs.test_repo_name }}
           ref: ${{ inputs.test_repo_branch }}
 
+      - name: Set prefix
+        id: set-prefix
+        run: |
+          COMMIT_SHA=$(git rev-parse --short HEAD)
+          echo "test_binaries_prefix=integration-test/test-binaries/${COMMIT_SHA}" >> "$GITHUB_OUTPUT"
+
       - name: Set up Go 1.x
         uses: actions/setup-go@v4
         with:
@@ -63,41 +72,37 @@ jobs:
           aws-region: us-west-2
 
       - name: Build and upload test binaries
-        id: build
         env:
           S3_BUCKET: ${{ inputs.s3_integration_bucket }}
+          GOARCH: ${{ matrix.arch }}
         run: |
           COMMIT_SHA=$(git rev-parse --short HEAD)
-          echo "test_binaries_prefix=integration-test/test-binaries/${COMMIT_SHA}" >> "$GITHUB_OUTPUT"
+          PREFIX="integration-test/test-binaries/${COMMIT_SHA}/linux/${GOARCH}"
+          mkdir -p "build/bin/${GOARCH}"
 
           go mod download
 
-          for GOARCH in amd64 arm64; do
-            PREFIX="integration-test/test-binaries/${COMMIT_SHA}/linux/${GOARCH}"
-            mkdir -p "build/bin/${GOARCH}"
-
-            echo "=== Building for linux/${GOARCH} ==="
-            for pkg in $(go list ./test/...); do
-              name=$(basename "$pkg")
-              if CGO_ENABLED=0 GOOS=linux GOARCH=$GOARCH go test -c -o "build/bin/${GOARCH}/${name}.test" "$pkg" 2>&1; then
-                echo "  ✓ ${name}.test"
-              else
-                echo "  ✗ ${name}.test (skipped)"
-              fi
-            done
-
-            BUILT=$(ls "build/bin/${GOARCH}/"*.test 2>/dev/null | wc -l)
-            echo "Built ${BUILT} binaries for linux/${GOARCH}"
-
-            aws s3 cp "build/bin/${GOARCH}/" "s3://${S3_BUCKET}/${PREFIX}/" --recursive --quiet
+          echo "=== Building for linux/${GOARCH} ==="
+          for pkg in $(go list ./test/...); do
+            name=$(basename "$pkg")
+            if CGO_ENABLED=0 GOOS=linux GOARCH=$GOARCH go test -c -o "build/bin/${GOARCH}/${name}.test" "$pkg" 2>&1; then
+              echo "  ✓ ${name}.test"
+            else
+              echo "  ✗ ${name}.test (skipped)"
+            fi
           done
+
+          BUILT=$(ls "build/bin/${GOARCH}/"*.test 2>/dev/null | wc -l)
+          echo "Built ${BUILT} binaries for linux/${GOARCH}"
+
+          aws s3 cp "build/bin/${GOARCH}/" "s3://${S3_BUCKET}/${PREFIX}/" --recursive --quiet
           echo "Upload complete to commercial bucket"
 
       - name: Upload binaries artifact
         uses: actions/upload-artifact@v4
         with:
-          name: test-binaries
-          path: build/bin/
+          name: test-binaries-${{ matrix.arch }}
+          path: build/bin/${{ matrix.arch }}/
           retention-days: 1
 
   UploadTestBinariesITAR:
@@ -109,11 +114,17 @@ jobs:
       id-token: write
       contents: read
     steps:
-      - name: Download binaries artifact
+      - name: Download amd64 binaries
+        uses: actions/download-artifact@v4
+        with:
+          name: test-binaries-amd64
+          path: build/bin/amd64/
+
+      - name: Download arm64 binaries
         uses: actions/download-artifact@v4
         with:
-          name: test-binaries
-          path: build/bin/
+          name: test-binaries-arm64
+          path: build/bin/arm64/
 
       - name: Configure AWS Credentials for ITAR
         uses: aws-actions/configure-aws-credentials@v4
@@ -140,11 +151,17 @@ jobs:
       id-token: write
       contents: read
     steps:
-      - name: Download binaries artifact
+      - name: Download amd64 binaries
+        uses: actions/download-artifact@v4
+        with:
+          name: test-binaries-amd64
+          path: build/bin/amd64/
+
+      - name: Download arm64 binaries
         uses: actions/download-artifact@v4
         with:
-          name: test-binaries
-          path: build/bin/
+          name: test-binaries-arm64
+          path: build/bin/arm64/
 
       - name: Configure AWS Credentials for CN
         uses: aws-actions/configure-aws-credentials@v4
diff --git a/.github/workflows/test-build-packages.yml b/.github/workflows/test-build-packages.yml
index 0814a80047..4eb7ec55b9 100644
--- a/.github/workflows/test-build-packages.yml
+++ b/.github/workflows/test-build-packages.yml
@@ -61,8 +61,17 @@ on:
 
 jobs:
   MakeMacPkg:
-    name: 'MakeMacPkg'
+    name: 'MakeMacPkg-${{ matrix.arch }}'
     runs-on: macos-15
+    strategy:
+      matrix:
+        include:
+          - arch: amd64
+            make_target: amazon-cloudwatch-agent-darwin-amd64 package-darwin-amd64
+            bin_dir: darwin/amd64
+          - arch: arm64
+            make_target: amazon-cloudwatch-agent-darwin-arm64 package-darwin-arm64
+            bin_dir: darwin/arm64
     permissions:
       id-token: write
       contents: read
@@ -96,7 +105,7 @@ jobs:
         id: cached_binaries
         uses: actions/cache@v3
         with:
-          key: "cached-binaries-${{ runner.os }}-${{ inputs.BucketKey }}"
+          key: "cached-binaries-${{ runner.os }}-${{ matrix.arch }}-${{ inputs.BucketKey }}"
           path: go.mod
 
       - name: Cache pkg
@@ -111,33 +120,29 @@ jobs:
       - name: Build Binaries
         if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false
         working-directory: cwa
-        run: make amazon-cloudwatch-agent-darwin package-darwin
+        run: make ${{ matrix.make_target }}
 
       - name: Copy binary
         if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false
         working-directory: cwa
         run: |
           echo cw agent version $(cat CWAGENT_VERSION)
-          cp -r build/bin/darwin/amd64/. /tmp/
-          cp -r build/bin/darwin/arm64/. /tmp/arm64/
-          cp build/bin/CWAGENT_VERSION /tmp/CWAGENT_VERSION
+          mkdir -p /tmp/${{ matrix.arch }}
+          cp -r build/bin/${{ matrix.bin_dir }}/. /tmp/${{ matrix.arch }}/
+          cp build/bin/CWAGENT_VERSION /tmp/${{ matrix.arch }}/CWAGENT_VERSION
 
       - name: Create pkg dep folder and copy deps
         if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false
         working-directory: test
         run: |
-          cp -r pkg/tools/. /tmp/
-          cp -r pkg/tools/. /tmp/arm64/
+          cp -r pkg/tools/. /tmp/${{ matrix.arch }}/
 
       - name: Build And Upload PKG
         if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false
-        working-directory: /tmp/
+        working-directory: /tmp/${{ matrix.arch }}
         run: |
           chmod +x create_pkg.sh
-          chmod +x arm64/create_pkg.sh
-          ./create_pkg.sh ${{ inputs.Bucket }}/${{ inputs.PackageBucketKey }} "nosha" amd64
-          cd arm64
-          ./create_pkg.sh ${{ inputs.Bucket }}/${{ inputs.PackageBucketKey }} "nosha" arm64
+          ./create_pkg.sh ${{ inputs.Bucket }}/${{ inputs.PackageBucketKey }} "nosha" ${{ matrix.arch }}
 
   #GH actions set up gpg only works on ubuntu as of this commit date
   GPGSignMacPackage:
diff --git a/Makefile b/Makefile
index 67599ba3a9..033de61adb 100644
--- a/Makefile
+++ b/Makefile
@@ -94,6 +94,30 @@ ifeq ($(shell uname -s),Darwin)
 endif
 endif
 
+amazon-cloudwatch-agent-darwin-amd64: copy-version-file workload-discovery-darwin-amd64
+ifneq ($(OS),Windows_NT)
+ifeq ($(shell uname -s),Darwin)
+	@echo Building CloudWatchAgent for MacOS AMD64
+	$(DARWIN_BUILD_AMD64)/config-downloader github.com/aws/amazon-cloudwatch-agent/cmd/config-downloader
+	$(DARWIN_BUILD_AMD64)/config-translator github.com/aws/amazon-cloudwatch-agent/cmd/config-translator
+	$(DARWIN_BUILD_AMD64)/amazon-cloudwatch-agent github.com/aws/amazon-cloudwatch-agent/cmd/amazon-cloudwatch-agent
+	$(DARWIN_BUILD_AMD64)/start-amazon-cloudwatch-agent github.com/aws/amazon-cloudwatch-agent/cmd/start-amazon-cloudwatch-agent
+	$(DARWIN_BUILD_AMD64)/amazon-cloudwatch-agent-config-wizard github.com/aws/amazon-cloudwatch-agent/cmd/amazon-cloudwatch-agent-config-wizard
+endif
+endif
+
+amazon-cloudwatch-agent-darwin-arm64: copy-version-file workload-discovery-darwin-arm64
+ifneq ($(OS),Windows_NT)
+ifeq ($(shell uname -s),Darwin)
+	@echo Building CloudWatchAgent for MacOS ARM64
+	$(DARWIN_BUILD_ARM64)/config-downloader github.com/aws/amazon-cloudwatch-agent/cmd/config-downloader
+	$(DARWIN_BUILD_ARM64)/config-translator github.com/aws/amazon-cloudwatch-agent/cmd/config-translator
+	$(DARWIN_BUILD_ARM64)/amazon-cloudwatch-agent github.com/aws/amazon-cloudwatch-agent/cmd/amazon-cloudwatch-agent
+	$(DARWIN_BUILD_ARM64)/start-amazon-cloudwatch-agent github.com/aws/amazon-cloudwatch-agent/cmd/start-amazon-cloudwatch-agent
+	$(DARWIN_BUILD_ARM64)/amazon-cloudwatch-agent-config-wizard github.com/aws/amazon-cloudwatch-agent/cmd/amazon-cloudwatch-agent-config-wizard
+endif
+endif
+
 amazon-cloudwatch-agent-windows: copy-version-file workload-discovery-windows
 	@echo Building CloudWatchAgent for Windows with AMD64
 	$(WIN_BUILD)/config-downloader.exe github.com/aws/amazon-cloudwatch-agent/cmd/config-downloader
@@ -114,6 +138,20 @@ ifeq ($(shell uname -s),Darwin)
 endif
 endif
 
+workload-discovery-darwin-amd64:
+ifneq ($(OS),Windows_NT)
+ifeq ($(shell uname -s),Darwin)
+	CGO_ENABLED=1 GO111MODULE=on GOOS=darwin GOARCH=amd64 go build -trimpath -ldflags="-s -w" -buildmode=${CWAGENT_BUILD_MODE} -o $(BUILD_SPACE)/bin/darwin_amd64/workload-discovery github.com/aws/amazon-cloudwatch-agent/cmd/workload-discovery
+endif
+endif
+
+workload-discovery-darwin-arm64:
+ifneq ($(OS),Windows_NT)
+ifeq ($(shell uname -s),Darwin)
+	CGO_ENABLED=1 GO111MODULE=on GOOS=darwin GOARCH=arm64 go build -trimpath -ldflags="-s -w" -buildmode=${CWAGENT_BUILD_MODE} -o $(BUILD_SPACE)/bin/darwin_arm64/workload-discovery github.com/aws/amazon-cloudwatch-agent/cmd/workload-discovery
+endif
+endif
+
 workload-discovery-windows:
 	GOOS=windows GOARCH=amd64 go build -trimpath -ldflags="-s -w" -buildmode=${CWAGENT_BUILD_MODE} -o $(BUILD_SPACE)/bin/windows_amd64/workload-discovery.exe github.com/aws/amazon-cloudwatch-agent/cmd/workload-discovery
 
@@ -337,6 +375,32 @@ package-prepare-darwin-tar:
 
 	cp -rf $(BASE_SPACE)/Tools $(BUILD_SPACE)/
 
+package-prepare-darwin-tar-amd64:
+	mkdir -p $(BUILD_SPACE)/private/darwin/amd64/tar/amazon-cloudwatch-agent-pre-pkg
+	cp $(BUILD_SPACE)/bin/darwin_amd64/* $(BUILD_SPACE)/private/darwin/amd64/tar/amazon-cloudwatch-agent-pre-pkg/
+	cp $(BASE_SPACE)/licensing/* $(BUILD_SPACE)/private/darwin/amd64/tar/amazon-cloudwatch-agent-pre-pkg/
+	cp $(BASE_SPACE)/RELEASE_NOTES $(BUILD_SPACE)/private/darwin/amd64/tar/amazon-cloudwatch-agent-pre-pkg/
+	cp $(BUILD_SPACE)/bin/CWAGENT_VERSION $(BUILD_SPACE)/private/darwin/amd64/tar/amazon-cloudwatch-agent-pre-pkg/
+	cp $(BASE_SPACE)/cfg/commonconfig/common-config.toml $(BUILD_SPACE)/private/darwin/amd64/tar/amazon-cloudwatch-agent-pre-pkg/
+	cp $(BASE_SPACE)/translator/config/schema.json $(BUILD_SPACE)/private/darwin/amd64/tar/amazon-cloudwatch-agent-pre-pkg/amazon-cloudwatch-agent-schema.json
+	cp $(BASE_SPACE)/packaging/darwin/amazon-cloudwatch-agent-ctl $(BUILD_SPACE)/private/darwin/amd64/tar/amazon-cloudwatch-agent-pre-pkg/
+	cp $(BASE_SPACE)/packaging/darwin/com.amazon.cloudwatch.agent.plist $(BUILD_SPACE)/private/darwin/amd64/tar/amazon-cloudwatch-agent-pre-pkg/
+	cp $(BASE_SPACE)/packaging/opentelemetry-jmx-metrics.jar $(BUILD_SPACE)/private/darwin/amd64/tar/amazon-cloudwatch-agent-pre-pkg/opentelemetry-jmx-metrics.jar
+	cp -rf $(BASE_SPACE)/Tools $(BUILD_SPACE)/
+
+package-prepare-darwin-tar-arm64:
+	mkdir -p $(BUILD_SPACE)/private/darwin/arm64/tar/amazon-cloudwatch-agent-pre-pkg
+	cp $(BUILD_SPACE)/bin/darwin_arm64/* $(BUILD_SPACE)/private/darwin/arm64/tar/amazon-cloudwatch-agent-pre-pkg/
+	cp $(BASE_SPACE)/licensing/* $(BUILD_SPACE)/private/darwin/arm64/tar/amazon-cloudwatch-agent-pre-pkg/
+	cp $(BASE_SPACE)/RELEASE_NOTES $(BUILD_SPACE)/private/darwin/arm64/tar/amazon-cloudwatch-agent-pre-pkg/
+	cp $(BUILD_SPACE)/bin/CWAGENT_VERSION $(BUILD_SPACE)/private/darwin/arm64/tar/amazon-cloudwatch-agent-pre-pkg/
+	cp $(BASE_SPACE)/cfg/commonconfig/common-config.toml $(BUILD_SPACE)/private/darwin/arm64/tar/amazon-cloudwatch-agent-pre-pkg/
+	cp $(BASE_SPACE)/translator/config/schema.json $(BUILD_SPACE)/private/darwin/arm64/tar/amazon-cloudwatch-agent-pre-pkg/amazon-cloudwatch-agent-schema.json
+	cp $(BASE_SPACE)/packaging/darwin/amazon-cloudwatch-agent-ctl $(BUILD_SPACE)/private/darwin/arm64/tar/amazon-cloudwatch-agent-pre-pkg/
+	cp $(BASE_SPACE)/packaging/darwin/com.amazon.cloudwatch.agent.plist $(BUILD_SPACE)/private/darwin/arm64/tar/amazon-cloudwatch-agent-pre-pkg/
+	cp $(BASE_SPACE)/packaging/opentelemetry-jmx-metrics.jar $(BUILD_SPACE)/private/darwin/arm64/tar/amazon-cloudwatch-agent-pre-pkg/opentelemetry-jmx-metrics.jar
+	cp -rf $(BASE_SPACE)/Tools $(BUILD_SPACE)/
+
 .PHONY: package-rpm
 package-rpm: package-prepare-rpm
 	ARCH=amd64 TARGET_SUPPORTED_ARCH=x86_64 PREPKGPATH="$(BUILD_SPACE)/private/linux/amd64/rpm/amazon-cloudwatch-agent-pre-pkg" $(BUILD_SPACE)/Tools/src/create_rpm.sh
@@ -356,6 +420,12 @@ package-darwin: package-prepare-darwin-tar
 	ARCH=amd64 TARGET_SUPPORTED_ARCH=x86_64 PREPKGPATH="$(BUILD_SPACE)/private/darwin/amd64/tar/amazon-cloudwatch-agent-pre-pkg" $(BUILD_SPACE)/Tools/src/create_darwin.sh
 	ARCH=arm64 TARGET_SUPPORTED_ARCH=aarch64 PREPKGPATH="$(BUILD_SPACE)/private/darwin/arm64/tar/amazon-cloudwatch-agent-pre-pkg" $(BUILD_SPACE)/Tools/src/create_darwin.sh
 
+package-darwin-amd64: package-prepare-darwin-tar-amd64
+	ARCH=amd64 TARGET_SUPPORTED_ARCH=x86_64 PREPKGPATH="$(BUILD_SPACE)/private/darwin/amd64/tar/amazon-cloudwatch-agent-pre-pkg" $(BUILD_SPACE)/Tools/src/create_darwin.sh
+
+package-darwin-arm64: package-prepare-darwin-tar-arm64
+	ARCH=arm64 TARGET_SUPPORTED_ARCH=aarch64 PREPKGPATH="$(BUILD_SPACE)/private/darwin/arm64/tar/amazon-cloudwatch-agent-pre-pkg" $(BUILD_SPACE)/Tools/src/create_darwin.sh
+
 .PHONY: fmt fmt-sh build test clean
 
 .PHONY: dockerized-build dockerized-build-vendor

From 1cbe231de9c6da3e8563924fa28db925e4bbef8d Mon Sep 17 00:00:00 2001
From: Marcus Mann <mpmann@amazon.com>
Date: Fri, 20 Feb 2026 12:15:15 -0500
Subject: [PATCH 12/19] fix: Skip matrix jobs when array is empty

Add if conditions to skip jobs when their matrix arrays are empty,
preventing 'Matrix vector does not contain any values' errors.
---
 .github/workflows/ec2-integration-test.yml |  1 +
 .github/workflows/test-artifacts.yml       | 12 ++++++++++++
 2 files changed, 13 insertions(+)

diff --git a/.github/workflows/ec2-integration-test.yml b/.github/workflows/ec2-integration-test.yml
index 985fc728fa..868074f1cc 100644
--- a/.github/workflows/ec2-integration-test.yml
+++ b/.github/workflows/ec2-integration-test.yml
@@ -55,6 +55,7 @@ on:
 
 jobs:
   EC2IntegrationTest:
+    if: ${{ inputs.test_props != '[]' }}
     name: ${{matrix.arrays.testName}}
     runs-on: ubuntu-latest
     strategy:
diff --git a/.github/workflows/test-artifacts.yml b/.github/workflows/test-artifacts.yml
index 64a4d6f98d..1bffae8d4e 100644
--- a/.github/workflows/test-artifacts.yml
+++ b/.github/workflows/test-artifacts.yml
@@ -408,6 +408,7 @@ jobs:
 
   EC2NvidiaGPUIntegrationTest:
     needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables ]
+    if: ${{ needs.GenerateTestMatrix.outputs.ec2_gpu_matrix != '[]' }}
     name: ${{ matrix.arrays.wip && '[WIP] ' || '' }}${{matrix.arrays.testName}}
     runs-on: ubuntu-latest
     strategy:
@@ -751,6 +752,7 @@ jobs:
 
   EC2WinIntegrationTest:
     needs: [OutputEnvVariables, GenerateTestMatrix]
+    if: ${{ needs.GenerateTestMatrix.outputs.ec2_windows_matrix != '[]' }}
     name:  ${{matrix.arrays.testName}}
     runs-on: ubuntu-latest
     strategy:
@@ -846,6 +848,7 @@ jobs:
             terraform destroy --auto-approve
   EC2DarwinIntegrationTest:
     needs: [GenerateTestMatrix, OutputEnvVariables]
+    if: ${{ needs.GenerateTestMatrix.outputs.ec2_mac_matrix != '[]' }}
     name: ${{ matrix.arrays.wip && '[WIP] ' || '' }}${{matrix.arrays.testName}}
     runs-on: ubuntu-latest
     strategy:
@@ -999,6 +1002,7 @@ jobs:
     name: ${{ matrix.arrays.wip && '[WIP] ' || '' }}${{matrix.arrays.testName}}
     runs-on: ubuntu-latest
     needs: [ GenerateTestMatrix, OutputEnvVariables ]
+    if: ${{ needs.GenerateTestMatrix.outputs.ecs_ec2_launch_daemon_matrix != '[]' }}
     strategy:
       fail-fast: false
       matrix:
@@ -1077,6 +1081,7 @@ jobs:
     name: ${{ matrix.arrays.wip && '[WIP] ' || '' }}${{matrix.arrays.testName}}
     runs-on: ubuntu-latest
     needs: [ GenerateTestMatrix, OutputEnvVariables ]
+    if: ${{ needs.GenerateTestMatrix.outputs.ecs_fargate_matrix != '[]' }}
     strategy:
       fail-fast: false
       matrix:
@@ -1167,6 +1172,7 @@ jobs:
     name: ${{ matrix.arrays.wip && '[WIP] ' || '' }}${{matrix.arrays.testName}}
     runs-on: ubuntu-latest
     needs: [ GenerateTestMatrix, OutputEnvVariables ]
+    if: ${{ needs.GenerateTestMatrix.outputs.eks_daemon_matrix != '[]' }}
     strategy:
       fail-fast: false
       matrix:
@@ -1260,6 +1266,7 @@ jobs:
     name: ${{ matrix.arrays.wip && '[WIP] ' || '' }}${{matrix.arrays.testName}}
     runs-on: ubuntu-latest
     needs: [ GenerateTestMatrix, OutputEnvVariables ]
+    if: ${{ needs.GenerateTestMatrix.outputs.eks_deployment_matrix != '[]' }}
     strategy:
       fail-fast: false
       matrix:
@@ -1349,6 +1356,7 @@ jobs:
   PerformanceTrackingTest:
     name: ${{ matrix.arrays.wip && '[WIP] ' || '' }}${{matrix.arrays.testName}}
     needs: [ GenerateTestMatrix, OutputEnvVariables ]
+    if: ${{ needs.GenerateTestMatrix.outputs.ec2_performance_matrix != '[]' }}
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -1430,6 +1438,7 @@ jobs:
   EC2WinPerformanceTest:
     name: ${{ matrix.arrays.wip && '[WIP] ' || '' }}${{matrix.arrays.testName}}
     needs: [ GenerateTestMatrix, OutputEnvVariables ]
+    if: ${{ needs.GenerateTestMatrix.outputs.ec2_windows_performance_matrix != '[]' }}
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -1511,6 +1520,7 @@ jobs:
   StressTrackingTest:
     name: ${{ matrix.arrays.wip && '[WIP] ' || '' }}${{matrix.arrays.testName}}
     needs: [GenerateTestMatrix, OutputEnvVariables ]
+    if: ${{ needs.GenerateTestMatrix.outputs.ec2_stress_matrix != '[]' }}
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -1593,6 +1603,7 @@ jobs:
   EC2WinStressTrackingTest:
     name: ${{ matrix.arrays.wip && '[WIP] ' || '' }}${{matrix.arrays.testName}}
     needs: [GenerateTestMatrix, OutputEnvVariables]
+    if: ${{ needs.GenerateTestMatrix.outputs.ec2_windows_stress_matrix != '[]' }}
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -1676,6 +1687,7 @@ jobs:
   GPUEndToEndTest:
     name:  ${{matrix.arrays.testName}}
     needs: [ GenerateTestMatrix, OutputEnvVariables ]
+    if: ${{ needs.GenerateTestMatrix.outputs.eks_addon_matrix != '[]' }}
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false

From 0a4348363dc5e4382acf677f171728fa190e47cb Mon Sep 17 00:00:00 2001
From: Marcus Mann <mpmann@amazon.com>
Date: Fri, 20 Feb 2026 12:17:35 -0500
Subject: [PATCH 13/19] fix: Use /tmp/ directly for MakeMacPkg since each arch
 runs on separate runner

---
 .github/workflows/test-build-packages.yml | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test-build-packages.yml b/.github/workflows/test-build-packages.yml
index 4eb7ec55b9..3bb34b348d 100644
--- a/.github/workflows/test-build-packages.yml
+++ b/.github/workflows/test-build-packages.yml
@@ -127,19 +127,18 @@ jobs:
         working-directory: cwa
         run: |
           echo cw agent version $(cat CWAGENT_VERSION)
-          mkdir -p /tmp/${{ matrix.arch }}
-          cp -r build/bin/${{ matrix.bin_dir }}/. /tmp/${{ matrix.arch }}/
-          cp build/bin/CWAGENT_VERSION /tmp/${{ matrix.arch }}/CWAGENT_VERSION
+          cp -r build/bin/${{ matrix.bin_dir }}/. /tmp/
+          cp build/bin/CWAGENT_VERSION /tmp/CWAGENT_VERSION
 
       - name: Create pkg dep folder and copy deps
         if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false
         working-directory: test
         run: |
-          cp -r pkg/tools/. /tmp/${{ matrix.arch }}/
+          cp -r pkg/tools/. /tmp/
 
       - name: Build And Upload PKG
         if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false
-        working-directory: /tmp/${{ matrix.arch }}
+        working-directory: /tmp
         run: |
           chmod +x create_pkg.sh
           ./create_pkg.sh ${{ inputs.Bucket }}/${{ inputs.PackageBucketKey }} "nosha" ${{ matrix.arch }}

From 9c15a5654914cd28385df968913d841a59debd62 Mon Sep 17 00:00:00 2001
From: Marcus Mann <mpmann@amazon.com>
Date: Fri, 20 Feb 2026 12:24:52 -0500
Subject: [PATCH 14/19] fix: Use separate go caches for mac amd64/arm64

---
 .github/workflows/ec2-integration-test.yml | 2 +-
 .github/workflows/test-build-packages.yml  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ec2-integration-test.yml b/.github/workflows/ec2-integration-test.yml
index 868074f1cc..432df903ce 100644
--- a/.github/workflows/ec2-integration-test.yml
+++ b/.github/workflows/ec2-integration-test.yml
@@ -136,7 +136,7 @@ jobs:
               -var="ssh_key_value=${{env.PRIVATE_KEY}}" \
               -var="test_dir=${{ matrix.arrays.test_dir }}" \
               -var="test_name=${{ matrix.arrays.os }}" \
-              -var="test_func=${{ matrix.arrays.testFunc }}" \
+              -var="test_run_filter=${{ matrix.arrays.testRunFilter }}" \
               -var="is_onprem=${{ inputs.is_onprem_test }}" \
               $TEST_BINARIES_VAR \
               -var="user=${{ matrix.arrays.username }}"; then
diff --git a/.github/workflows/test-build-packages.yml b/.github/workflows/test-build-packages.yml
index 3bb34b348d..ec795dd2bf 100644
--- a/.github/workflows/test-build-packages.yml
+++ b/.github/workflows/test-build-packages.yml
@@ -115,7 +115,7 @@ jobs:
           path: |
             ~/Library/Caches/go-build
             ~/go/pkg/mod
-          key: v1-go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.sum') }}
+          key: v1-go-pkg-mod-${{ runner.os }}-${{ matrix.arch }}-${{ hashFiles('**/go.sum') }}
 
       - name: Build Binaries
         if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false

From d956795ddbd88f755e53af5a42df144c901060b5 Mon Sep 17 00:00:00 2001
From: Marcus Mann <mpmann@amazon.com>
Date: Thu, 26 Feb 2026 14:34:04 -0500
Subject: [PATCH 15/19] fix: avoid 1MB job output limit by writing
 ec2_linux_matrix to file

The splitByTestFunc feature increased ec2_linux_matrix from ~306 to 939
entries (729KB). Combined with other matrices, total job outputs exceeded
GitHub's 1MB limit.

Write ec2_linux_matrix to a temp file instead of GITHUB_OUTPUT, then
paginate from the file. Only the paginated pages are output as job outputs.
---
 .github/workflows/test-artifacts.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test-artifacts.yml b/.github/workflows/test-artifacts.yml
index dc0de8d942..a183861afc 100644
--- a/.github/workflows/test-artifacts.yml
+++ b/.github/workflows/test-artifacts.yml
@@ -211,7 +211,8 @@ jobs:
           
           echo "ec2_gpu_matrix=$(apply_filters generator/resources/ec2_gpu_complete_test_matrix.json)" >> "$GITHUB_OUTPUT"
           echo "eks_addon_matrix=$(apply_filters generator/resources/eks_addon_complete_test_matrix.json)" >> "$GITHUB_OUTPUT"
-          echo "ec2_linux_matrix=$(apply_filters generator/resources/ec2_linux_complete_test_matrix.json)" >> "$GITHUB_OUTPUT"
+          # ec2_linux_matrix written to file (not GITHUB_OUTPUT) to avoid 1MB job output limit
+          apply_filters generator/resources/ec2_linux_complete_test_matrix.json > /tmp/ec2_linux_matrix.json
           echo "ec2_linux_onprem_matrix=$(apply_filters generator/resources/ec2_linux_onprem_complete_test_matrix.json)" >> "$GITHUB_OUTPUT"
           echo "ec2_selinux_matrix=$(apply_filters generator/resources/ec2_selinux_complete_test_matrix.json)" >> "$GITHUB_OUTPUT"
           echo "ec2_windows_matrix=$(apply_filters generator/resources/ec2_windows_complete_test_matrix.json)" >> "$GITHUB_OUTPUT"
@@ -233,7 +234,7 @@ jobs:
           # GitHub Actions matrix limit is 256 jobs per workflow run.
           # Use 200 per page for headroom. Up to 5 pages supported (1000 tests).
           PAGE_SIZE=200
-          FULL_MATRIX='${{ steps.set-matrix.outputs.ec2_linux_matrix }}'
+          FULL_MATRIX=$(cat /tmp/ec2_linux_matrix.json)
           TOTAL=$(echo "$FULL_MATRIX" | jq 'length')
           PAGE_COUNT=$(( (TOTAL + PAGE_SIZE - 1) / PAGE_SIZE ))
 

From 88210e66e6cb0c6751b684e9a601d372f7c0ed02 Mon Sep 17 00:00:00 2001
From: Marcus Mann <mpmann@amazon.com>
Date: Thu, 26 Feb 2026 15:05:19 -0500
Subject: [PATCH 16/19] fix: strip empty fields from test matrix to avoid 1MB
 output limit

The splitByTestFunc feature increased matrix size significantly. Strip
empty/zero/false/null fields from matrix JSON, reducing total output
size from ~935KB to ~573KB (well under GitHub's 1MB limit).
---
 .github/workflows/test-artifacts.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test-artifacts.yml b/.github/workflows/test-artifacts.yml
index a183861afc..08d7567be9 100644
--- a/.github/workflows/test-artifacts.yml
+++ b/.github/workflows/test-artifacts.yml
@@ -205,8 +205,8 @@ jobs:
               matrix_content=$(echo "$matrix_content" | jq -c '[.[] | select(.test_dir == "${{ inputs.test_dir_filter }}")]')
             fi
             
-            # Output compact JSON (single line) for GITHUB_OUTPUT compatibility
-            echo "$matrix_content" | jq -c '.'
+            # Strip empty/zero/false fields to reduce size (~40% smaller), then output compact JSON
+            echo "$matrix_content" | jq -c '[.[] | with_entries(select(.value != "" and .value != 0 and .value != false and .value != null))]'
           }
           
           echo "ec2_gpu_matrix=$(apply_filters generator/resources/ec2_gpu_complete_test_matrix.json)" >> "$GITHUB_OUTPUT"

From 76312d4d44beeacbc8abfd7d6873f0f8e4586b56 Mon Sep 17 00:00:00 2001
From: Marcus Mann <mpmann@amazon.com>
Date: Thu, 26 Feb 2026 15:08:57 -0500
Subject: [PATCH 17/19] Revert "fix: strip empty fields from test matrix to
 avoid 1MB output limit"

This reverts commit 88210e66e6cb0c6751b684e9a601d372f7c0ed02.
---
 .github/workflows/test-artifacts.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test-artifacts.yml b/.github/workflows/test-artifacts.yml
index 08d7567be9..a183861afc 100644
--- a/.github/workflows/test-artifacts.yml
+++ b/.github/workflows/test-artifacts.yml
@@ -205,8 +205,8 @@ jobs:
               matrix_content=$(echo "$matrix_content" | jq -c '[.[] | select(.test_dir == "${{ inputs.test_dir_filter }}")]')
             fi
             
-            # Strip empty/zero/false fields to reduce size (~40% smaller), then output compact JSON
-            echo "$matrix_content" | jq -c '[.[] | with_entries(select(.value != "" and .value != 0 and .value != false and .value != null))]'
+            # Output compact JSON (single line) for GITHUB_OUTPUT compatibility
+            echo "$matrix_content" | jq -c '.'
           }
           
           echo "ec2_gpu_matrix=$(apply_filters generator/resources/ec2_gpu_complete_test_matrix.json)" >> "$GITHUB_OUTPUT"

From bc99e3fc9f206405f29bb4352e0fa3c866ebb97d Mon Sep 17 00:00:00 2001
From: Marcus Mann <mpmann@amazon.com>
Date: Thu, 26 Feb 2026 15:58:49 -0500
Subject: [PATCH 18/19] fix: strip empty fields from matrix JSON to stay under
 1MB limit

Even with omitempty in the generator, total output size (~648KB) plus
overhead can exceed GitHub's 1MB job output limit. Add jq filtering
as additional safety to strip any remaining empty/zero/false fields.
---
 .github/workflows/test-artifacts.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test-artifacts.yml b/.github/workflows/test-artifacts.yml
index a183861afc..feea2d92de 100644
--- a/.github/workflows/test-artifacts.yml
+++ b/.github/workflows/test-artifacts.yml
@@ -205,8 +205,8 @@ jobs:
               matrix_content=$(echo "$matrix_content" | jq -c '[.[] | select(.test_dir == "${{ inputs.test_dir_filter }}")]')
             fi
             
-            # Output compact JSON (single line) for GITHUB_OUTPUT compatibility
-            echo "$matrix_content" | jq -c '.'
+            # Strip empty/zero/false fields to reduce size, then output compact JSON
+            echo "$matrix_content" | jq -c '[.[] | with_entries(select(.value != "" and .value != 0 and .value != false and .value != null))]'
           }
           
           echo "ec2_gpu_matrix=$(apply_filters generator/resources/ec2_gpu_complete_test_matrix.json)" >> "$GITHUB_OUTPUT"

From df53dc8e63a17f28cfae34f8f4ebf8e6f70dcb85 Mon Sep 17 00:00:00 2001
From: Marcus Mann <mpmann@amazon.com>
Date: Fri, 27 Feb 2026 14:22:40 -0500
Subject: [PATCH 19/19] fix: use artifacts for ec2_linux matrices to avoid 1MB
 output limit

Upload ec2_linux matrix pages as artifacts instead of job outputs.
Add loader jobs that download the artifact and output each page.
This completely bypasses the 1MB job output limit for GenerateTestMatrix.
---
 .github/workflows/test-artifacts.yml | 124 ++++++++++++++++++++-------
 1 file changed, 91 insertions(+), 33 deletions(-)

diff --git a/.github/workflows/test-artifacts.yml b/.github/workflows/test-artifacts.yml
index feea2d92de..3efcd46abc 100644
--- a/.github/workflows/test-artifacts.yml
+++ b/.github/workflows/test-artifacts.yml
@@ -153,11 +153,6 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       ec2_gpu_matrix: ${{ steps.set-matrix.outputs.ec2_gpu_matrix }}
-      ec2_linux_matrix_0: ${{ steps.paginate-matrix.outputs.ec2_linux_matrix_0 }}
-      ec2_linux_matrix_1: ${{ steps.paginate-matrix.outputs.ec2_linux_matrix_1 }}
-      ec2_linux_matrix_2: ${{ steps.paginate-matrix.outputs.ec2_linux_matrix_2 }}
-      ec2_linux_matrix_3: ${{ steps.paginate-matrix.outputs.ec2_linux_matrix_3 }}
-      ec2_linux_matrix_4: ${{ steps.paginate-matrix.outputs.ec2_linux_matrix_4 }}
       ec2_linux_matrix_page_count: ${{ steps.paginate-matrix.outputs.ec2_linux_matrix_page_count }}
       ec2_linux_onprem_matrix: ${{ steps.set-matrix.outputs.ec2_linux_onprem_matrix }}
       ec2_selinux_matrix: ${{ steps.set-matrix.outputs.ec2_selinux_matrix }}
@@ -246,29 +241,27 @@ jobs:
           echo "ec2_linux total=$TOTAL pages=$PAGE_COUNT"
           echo "ec2_linux_matrix_page_count=${PAGE_COUNT}" >> "$GITHUB_OUTPUT"
 
+          mkdir -p /tmp/ec2_linux_matrices
           for (( i=0; i<PAGE_COUNT; i++ )); do
             START=$(( i * PAGE_SIZE ))
-            PAGE=$(echo "$FULL_MATRIX" | jq -c ".[$START:$START+$PAGE_SIZE]")
-            echo "ec2_linux_matrix_${i}=${PAGE}" >> "$GITHUB_OUTPUT"
-            echo "  page $i: $(echo "$PAGE" | jq 'length') entries"
+            echo "$FULL_MATRIX" | jq -c ".[$START:$START+$PAGE_SIZE]" > "/tmp/ec2_linux_matrices/ec2_linux_matrix_${i}.json"
+            echo "  page $i: $(jq 'length' /tmp/ec2_linux_matrices/ec2_linux_matrix_${i}.json) entries"
           done
 
+      - name: Upload ec2_linux matrices
+        uses: actions/upload-artifact@v4
+        with:
+          name: ec2-linux-matrices
+          path: /tmp/ec2_linux_matrices/
+          retention-days: 1
+
       - name: Echo test plan matrix
         run: |
           echo "ec2_gpu_matrix: ${{ steps.set-matrix.outputs.ec2_gpu_matrix }}"
           echo "eks_addon_matrix: ${{ steps.set-matrix.outputs.eks_addon_matrix }}"
           echo "ec2_linux_matrix pages: ${{ steps.paginate-matrix.outputs.ec2_linux_matrix_page_count }}"
-          for i in 0 1 2 3 4; do
-            page='${{ steps.paginate-matrix.outputs.ec2_linux_matrix_0 }}'
-            case $i in
-              1) page='${{ steps.paginate-matrix.outputs.ec2_linux_matrix_1 }}' ;;
-              2) page='${{ steps.paginate-matrix.outputs.ec2_linux_matrix_2 }}' ;;
-              3) page='${{ steps.paginate-matrix.outputs.ec2_linux_matrix_3 }}' ;;
-              4) page='${{ steps.paginate-matrix.outputs.ec2_linux_matrix_4 }}' ;;
-            esac
-            if [ -n "$page" ]; then
-              echo "ec2_linux_matrix_$i: $(echo "$page" | jq 'length') entries"
-            fi
+          for f in /tmp/ec2_linux_matrices/*.json; do
+            echo "$(basename $f): $(jq 'length' $f) entries"
           done
           echo "ec2_linux_onprem_matrix: ${{ steps.set-matrix.outputs.ec2_linux_onprem_matrix }}"
           echo "ec2_selinux_matrix: ${{ steps.set-matrix.outputs.ec2_selinux_matrix }}"
@@ -559,15 +552,80 @@ jobs:
             fi
             terraform destroy --auto-approve
 
+  # Loader jobs to read ec2_linux matrices from artifact (avoids 1MB job output limit)
+  LoadEC2LinuxMatrix-0:
+    needs: [GenerateTestMatrix]
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.load.outputs.matrix }}
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: ec2-linux-matrices
+      - id: load
+        run: echo "matrix=$(cat ec2_linux_matrix_0.json)" >> "$GITHUB_OUTPUT"
+
+  LoadEC2LinuxMatrix-1:
+    needs: [GenerateTestMatrix]
+    if: ${{ needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 1 }}
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.load.outputs.matrix }}
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: ec2-linux-matrices
+      - id: load
+        run: echo "matrix=$(cat ec2_linux_matrix_1.json)" >> "$GITHUB_OUTPUT"
+
+  LoadEC2LinuxMatrix-2:
+    needs: [GenerateTestMatrix]
+    if: ${{ needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 2 }}
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.load.outputs.matrix }}
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: ec2-linux-matrices
+      - id: load
+        run: echo "matrix=$(cat ec2_linux_matrix_2.json)" >> "$GITHUB_OUTPUT"
+
+  LoadEC2LinuxMatrix-3:
+    needs: [GenerateTestMatrix]
+    if: ${{ needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 3 }}
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.load.outputs.matrix }}
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: ec2-linux-matrices
+      - id: load
+        run: echo "matrix=$(cat ec2_linux_matrix_3.json)" >> "$GITHUB_OUTPUT"
+
+  LoadEC2LinuxMatrix-4:
+    needs: [GenerateTestMatrix]
+    if: ${{ needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 4 }}
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.load.outputs.matrix }}
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: ec2-linux-matrices
+      - id: load
+        run: echo "matrix=$(cat ec2_linux_matrix_4.json)" >> "$GITHUB_OUTPUT"
+
   EC2LinuxIntegrationTest-0:
-    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, BuildTestBinaries ]
+    needs: [ StartLocalStack, LoadEC2LinuxMatrix-0, OutputEnvVariables, BuildTestBinaries ]
     name: 'EC2Linux-0'
     uses:  ./.github/workflows/ec2-integration-test.yml
     with:
       build_id: ${{ inputs.build_id }}
       test_dir: terraform/ec2/linux
       job_id: ec2-linux-integration-test-0
-      test_props: ${{ needs.GenerateTestMatrix.outputs.ec2_linux_matrix_0 }}
+      test_props: ${{ needs.LoadEC2LinuxMatrix-0.outputs.matrix }}
       test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }}
       test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }}
       test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}
@@ -581,15 +639,15 @@ jobs:
     secrets: inherit
 
   EC2LinuxIntegrationTest-1:
-    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, BuildTestBinaries ]
-    if: ${{ needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 1 }}
+    needs: [ StartLocalStack, LoadEC2LinuxMatrix-1, OutputEnvVariables, BuildTestBinaries ]
+    if: ${{ needs.LoadEC2LinuxMatrix-1.result == 'success' }}
     name: 'EC2Linux-1'
     uses:  ./.github/workflows/ec2-integration-test.yml
     with:
       build_id: ${{ inputs.build_id }}
       test_dir: terraform/ec2/linux
       job_id: ec2-linux-integration-test-1
-      test_props: ${{ needs.GenerateTestMatrix.outputs.ec2_linux_matrix_1 }}
+      test_props: ${{ needs.LoadEC2LinuxMatrix-1.outputs.matrix }}
       test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }}
       test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }}
       test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}
@@ -603,15 +661,15 @@ jobs:
     secrets: inherit
 
   EC2LinuxIntegrationTest-2:
-    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, BuildTestBinaries ]
-    if: ${{ needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 2 }}
+    needs: [ StartLocalStack, LoadEC2LinuxMatrix-2, OutputEnvVariables, BuildTestBinaries ]
+    if: ${{ needs.LoadEC2LinuxMatrix-2.result == 'success' }}
     name: 'EC2Linux-2'
     uses:  ./.github/workflows/ec2-integration-test.yml
     with:
       build_id: ${{ inputs.build_id }}
       test_dir: terraform/ec2/linux
       job_id: ec2-linux-integration-test-2
-      test_props: ${{ needs.GenerateTestMatrix.outputs.ec2_linux_matrix_2 }}
+      test_props: ${{ needs.LoadEC2LinuxMatrix-2.outputs.matrix }}
       test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }}
       test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }}
       test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}
@@ -625,15 +683,15 @@ jobs:
     secrets: inherit
 
   EC2LinuxIntegrationTest-3:
-    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, BuildTestBinaries ]
-    if: ${{ needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 3 }}
+    needs: [ StartLocalStack, LoadEC2LinuxMatrix-3, OutputEnvVariables, BuildTestBinaries ]
+    if: ${{ needs.LoadEC2LinuxMatrix-3.result == 'success' }}
     name: 'EC2Linux-3'
     uses:  ./.github/workflows/ec2-integration-test.yml
     with:
       build_id: ${{ inputs.build_id }}
       test_dir: terraform/ec2/linux
       job_id: ec2-linux-integration-test-3
-      test_props: ${{ needs.GenerateTestMatrix.outputs.ec2_linux_matrix_3 }}
+      test_props: ${{ needs.LoadEC2LinuxMatrix-3.outputs.matrix }}
       test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }}
       test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }}
       test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}
@@ -647,15 +705,15 @@ jobs:
     secrets: inherit
 
   EC2LinuxIntegrationTest-4:
-    needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables, BuildTestBinaries ]
-    if: ${{ needs.GenerateTestMatrix.outputs.ec2_linux_matrix_page_count > 4 }}
+    needs: [ StartLocalStack, LoadEC2LinuxMatrix-4, OutputEnvVariables, BuildTestBinaries ]
+    if: ${{ needs.LoadEC2LinuxMatrix-4.result == 'success' }}
     name: 'EC2Linux-4'
     uses:  ./.github/workflows/ec2-integration-test.yml
     with:
       build_id: ${{ inputs.build_id }}
       test_dir: terraform/ec2/linux
       job_id: ec2-linux-integration-test-4
-      test_props: ${{ needs.GenerateTestMatrix.outputs.ec2_linux_matrix_4 }}
+      test_props: ${{ needs.LoadEC2LinuxMatrix-4.outputs.matrix }}
       test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }}
       test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }}
       test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}