ci: setup ARM64 builds with CGO_ENABLED=1 compatibility

Signed-off-by: Doug Edgar <dedgar@redhat.com>

Author: rhdedgar

.github/workflows/build-image.yml

@@ -8,10 +8,23 @@ permissions:
   actions: write
   contents: read
 
+env:
+  IMG: quay.io/llamastack/llama-stack-k8s-operator:latest
+
 jobs:
-  build-latest-image:
+  build-arch:
     if: github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'main'
-    runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: amd64
+            runner: ubuntu-24.04
+            platform: linux/amd64
+          - arch: arm64
+            runner: ubuntu-24.04-arm
+            platform: linux/arm64
+    runs-on: ${{ matrix.runner }}
     steps:
       - name: Checkout code
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -31,5 +44,30 @@ jobs:
           username: ${{ secrets.APP_QUAY_USERNAME }}
           password: ${{ secrets.APP_QUAY_TOKEN }}
 
-      - name: Build and push multi-arch image
-        run: make image-buildx -e IMG=quay.io/llamastack/llama-stack-k8s-operator:latest
+      - name: Build and push single-arch image
+        run: |
+          make image-build-push-single \
+            CONTAINER_TOOL=docker \
+            PLATFORM=${{ matrix.platform }} \
+            IMG=${{ env.IMG }}-${{ matrix.arch }}
+
+  create-manifest:
+    if: github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'main'
+    needs: build-arch
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f
+
+      - name: Login to Quay.io
+        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
+        with:
+          registry: quay.io
+          username: ${{ secrets.APP_QUAY_USERNAME }}
+          password: ${{ secrets.APP_QUAY_TOKEN }}
+
+      - name: Create and push multi-arch manifest
+        run: |
+          docker buildx imagetools create -t ${{ env.IMG }} \
+            ${{ env.IMG }}-amd64 \
+            ${{ env.IMG }}-arm64

.github/workflows/generate-release.yml

@@ -165,9 +165,6 @@ jobs:
         run: |
           make test
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f
-
       - name: Check E2E Test Results
         run: |
           echo "E2E Test Results:"
@@ -183,10 +180,10 @@ jobs:
             echo "E2E tests passed - proceeding with release"
           fi
 
-      - name: Validate build release image
+      - name: Validate release image build
         shell: bash
         run: |
-          make image-buildx-build IMG=quay.io/llamastack/llama-stack-k8s-operator:v${{ steps.validate.outputs.operator_version }}
+          make image-build IMG=quay.io/llamastack/llama-stack-k8s-operator:v${{ steps.validate.outputs.operator_version }}
 
       - name: Commit and push changes
         shell: bash
@@ -203,13 +200,58 @@ jobs:
             echo "✅ Committed release changes to ${{ steps.validate.outputs.release_branch }}"
           fi
 
-  finalize-release:
+  build-release-arch:
     needs: generate-release
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: amd64
+            runner: ubuntu-24.04
+            platform: linux/amd64
+          - arch: arm64
+            runner: ubuntu-24.04-arm
+            platform: linux/arm64
+    runs-on: ${{ matrix.runner }}
+    env:
+      operator_version: ${{ needs.generate-release.outputs.operator_version }}
+      release_branch: ${{ needs.generate-release.outputs.release_branch }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+        with:
+          ref: ${{ env.release_branch }}
+
+      - name: Set up Go
+        uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
+        with:
+          go-version-file: go.mod
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f
+
+      - name: Log in to Quay.io
+        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
+        with:
+          registry: quay.io
+          username: ${{ secrets.APP_QUAY_USERNAME }}
+          password: ${{ secrets.APP_QUAY_TOKEN }}
+
+      - name: Build and push single-arch release image
+        run: |
+          make image-build-push-single \
+            CONTAINER_TOOL=docker \
+            PLATFORM=${{ matrix.platform }} \
+            IMG=quay.io/llamastack/llama-stack-k8s-operator:v${{ env.operator_version }}-${{ matrix.arch }}
+
+  finalize-release:
+    needs: [generate-release, build-release-arch]
     runs-on: ubuntu-24.04
     env:
       operator_version: ${{ needs.generate-release.outputs.operator_version }}
       llamastack_version: ${{ needs.generate-release.outputs.llamastack_version }}
       release_branch: ${{ needs.generate-release.outputs.release_branch }}
+      IMG: quay.io/llamastack/llama-stack-k8s-operator
     steps:
       - name: Checkout code
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
@@ -268,11 +310,6 @@ jobs:
         env:
           GH_TOKEN: ${{ github.token }}
 
-      - name: Set up Go
-        uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
-        with:
-          go-version-file: go.mod
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f
 
@@ -283,17 +320,20 @@ jobs:
           username: ${{ secrets.APP_QUAY_USERNAME }}
           password: ${{ secrets.APP_QUAY_TOKEN }}
 
-      - name: Build and push multi-arch release image
+      - name: Create and push multi-arch manifest
         shell: bash
         run: |
-          make image-buildx IMG=quay.io/llamastack/llama-stack-k8s-operator:v${{ env.operator_version }}
-          echo "✅ Pushed multi-arch image: quay.io/llamastack/llama-stack-k8s-operator:v${{ env.operator_version }}"
+          docker buildx imagetools create \
+            -t ${{ env.IMG }}:v${{ env.operator_version }} \
+            ${{ env.IMG }}:v${{ env.operator_version }}-amd64 \
+            ${{ env.IMG }}:v${{ env.operator_version }}-arm64
+          echo "✅ Pushed multi-arch image: ${{ env.IMG }}:v${{ env.operator_version }}"
 
       - name: Release complete
         shell: bash
         run: |
           echo "🚀 Release v${{ env.operator_version }} completed successfully!"
-          echo "📦 Container image: quay.io/llamastack/llama-stack-k8s-operator:v${{ env.operator_version }}"
+          echo "📦 Container image: ${{ env.IMG }}:v${{ env.operator_version }}"
           echo "📝 Release notes: https://github.com/${{ github.repository }}/releases/tag/v${{ env.operator_version }}"
           echo "🔧 Install with: kubectl apply -f https://raw.githubusercontent.com/llamastack/llama-stack-k8s-operator/v${{ env.operator_version }}/release/operator.yaml"
           echo "🌿 Release branch: ${{ env.release_branch }}"

.github/workflows/release-image.yml

@@ -16,9 +16,22 @@ permissions:
   actions: write
   contents: read
 
+env:
+  IMG: quay.io/llamastack/llama-stack-k8s-operator:${{ github.event.inputs.version }}
+
 jobs:
-  build-release-image:
-    runs-on: ubuntu-24.04
+  build-arch:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: amd64
+            runner: ubuntu-24.04
+            platform: linux/amd64
+          - arch: arm64
+            runner: ubuntu-24.04-arm
+            platform: linux/arm64
+    runs-on: ${{ matrix.runner }}
     steps:
       - name: Set up Go
         uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
@@ -40,6 +53,29 @@ jobs:
           username: ${{ secrets.APP_QUAY_USERNAME }}
           password: ${{ secrets.APP_QUAY_TOKEN }}
 
-      - name: Build and push multi-arch release image
+      - name: Build and push single-arch release image
+        run: |
+          make image-build-push-single \
+            CONTAINER_TOOL=docker \
+            PLATFORM=${{ matrix.platform }} \
+            IMG=${{ env.IMG }}-${{ matrix.arch }}
+
+  create-manifest:
+    needs: build-arch
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f
+
+      - name: Login to Quay.io
+        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
+        with:
+          registry: quay.io
+          username: ${{ secrets.APP_QUAY_USERNAME }}
+          password: ${{ secrets.APP_QUAY_TOKEN }}
+
+      - name: Create and push multi-arch manifest
         run: |
-          make image-buildx -e IMG=quay.io/llamastack/llama-stack-k8s-operator:${{ github.event.inputs.version }}
+          docker buildx imagetools create -t ${{ env.IMG }} \
+            ${{ env.IMG }}-amd64 \
+            ${{ env.IMG }}-arm64

Makefile

@@ -227,6 +227,42 @@ else
 endif
 	@echo "Successfully built multi-arch image: ${IMG}"
 
+# PLATFORM defines the target platform for single-arch image builds used by CI matrix jobs.
+# Each architecture is built natively on its own runner (e.g., ARM64 on ARM64 runner),
+# ensuring CGO_ENABLED=1 with full OpenSSL FIPS support for all architectures.
+PLATFORM ?= linux/amd64
+
+.PHONY: image-build-push-single
+image-build-push-single: ## Build and push a single-arch image (set PLATFORM and IMG)
+	@echo "Building and pushing single-arch image for platform: $(PLATFORM)"
+ifeq ($(CONTAINER_TOOL),docker)
+	- $(CONTAINER_TOOL) buildx create --name x-builder 2>/dev/null || true
+	$(CONTAINER_TOOL) buildx use x-builder
+	$(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORM) --tag ${IMG} .
+else
+	$(CONTAINER_TOOL) build --platform $(PLATFORM) -t ${IMG} .
+	$(CONTAINER_TOOL) push ${IMG}
+endif
+	@echo "Successfully pushed single-arch image: ${IMG} ($(PLATFORM))"
+
+# ARCH_IMGS is a space-separated list of per-architecture image references
+# used to create a multi-arch manifest (e.g., "myregistry/myimage:v1-amd64 myregistry/myimage:v1-arm64").
+ARCH_IMGS ?=
+
+.PHONY: image-create-manifest
+image-create-manifest: ## Create and push a multi-arch manifest from per-arch images (set IMG and ARCH_IMGS)
+	@echo "Creating multi-arch manifest: ${IMG}"
+ifeq ($(CONTAINER_TOOL),docker)
+	- $(CONTAINER_TOOL) buildx create --name x-builder 2>/dev/null || true
+	$(CONTAINER_TOOL) buildx use x-builder
+	$(CONTAINER_TOOL) buildx imagetools create -t ${IMG} $(ARCH_IMGS)
+else
+	$(CONTAINER_TOOL) manifest rm ${IMG} 2>/dev/null || true
+	$(CONTAINER_TOOL) manifest create ${IMG} $(ARCH_IMGS)
+	$(CONTAINER_TOOL) manifest push ${IMG}
+endif
+	@echo "Successfully pushed multi-arch manifest: ${IMG}"
+
 .PHONY: image-build-arm
 image-build-arm: ## Build ARM64 image with the manager
 	$(CONTAINER_TOOL) build --platform linux/arm64 -t ${IMG} .

README.md

@@ -238,8 +238,9 @@ This will cause all LlamaStackDistribution resources using the `starter` distrib
 
   **Note**:
   - The `image-buildx` target works with both Docker and Podman. It will automatically detect which tool is being used.
-  - **Native cross-compilation**: The Dockerfile uses `--platform=$BUILDPLATFORM` to run Go compilation natively on the build host, avoiding QEMU emulation for the build process. This dramatically improves build speed and reliability. Only the minimal final stage (package installation) runs under QEMU for cross-platform builds.
-  - **FIPS adherence**: Native builds use `CGO_ENABLED=1` with full OpenSSL FIPS support. Cross-compiled builds use `CGO_ENABLED=0` with pure Go FIPS (via `GOEXPERIMENT=strictfipsruntime`). Both approaches are Designed for FIPS.
+  - **Native builds in CI**: CI workflows use a matrix strategy with native runners for each architecture (AMD64 and ARM64). Each architecture is built on its own runner, avoiding QEMU emulation entirely. Per-architecture images are pushed separately, then combined into a single multi-arch manifest list. This ensures `CGO_ENABLED=1` with full OpenSSL FIPS support for all architectures.
+  - **Local cross-compilation**: For local development, the Dockerfile uses `--platform=$BUILDPLATFORM` to run Go compilation natively on the build host. When cross-compiling (e.g., building ARM64 on an AMD64 host), `CGO_ENABLED=0` is used with pure Go FIPS (via `GOEXPERIMENT=strictfipsruntime`). Native local builds use `CGO_ENABLED=1` with full OpenSSL FIPS support.
+  - **FIPS adherence**: All CI-produced images use `CGO_ENABLED=1` with full OpenSSL FIPS support via native builds on architecture-matched runners.
   - For Docker: Multi-arch builds require Docker Buildx. Ensure Docker Buildx is set up:
 
     ```commandline
@@ -249,6 +250,19 @@ This will cause all LlamaStackDistribution resources using the `starter` distrib
   - For Podman: Podman 4.0+ supports `podman buildx` (experimental). If buildx is unavailable, the Makefile will automatically fall back to using podman's native manifest-based multi-arch build approach.
   - The resulting images are multi-arch manifest lists, which means Kubernetes will automatically select the correct architecture when pulling the image.
 
+  **CI Build Targets**:
+
+  The CI workflows use the following Makefile targets for the matrix-based build strategy:
+
+  ```commandline
+  # Build and push a single-arch image (used by each matrix job on its native runner)
+  make image-build-push-single PLATFORM=linux/amd64 IMG=quay.io/<username>/llama-stack-k8s-operator:<tag>-amd64
+
+  # Create a multi-arch manifest from per-arch images (used by the final manifest job)
+  make image-create-manifest IMG=quay.io/<username>/llama-stack-k8s-operator:<tag> \
+    ARCH_IMGS="quay.io/<username>/llama-stack-k8s-operator:<tag>-amd64 quay.io/<username>/llama-stack-k8s-operator:<tag>-arm64"
+  ```
+
 - Building ARM64-only images
 
   To build a single ARM64 image (useful for testing or ARM-native systems):