fix(LTX25-PHASE-RESIDUE): 92% of the phase table's un-named time was one region, and two wall-clock ratios were measuring it (#1536) #1328

Workflow file for this run

.github/workflows/containers.yml at 7a5983e

	name: containers

	# Published container images -- ENG-RELEASE-CONTAINERS, issue #170.
	# Design: .agents/specs/container-images.md
	#
	# Authority is separated by stage, exactly as release.yml separates it:
	#
	# plan contents: read resolve version and lanes, validate the tag
	# verify contents: read build + validate every lane x arch. NO registry
	# write exists in this job, so a pull request
	# cannot publish even if a step tried to.
	# publish packages: write tags only. Builds, validates the image it is
	# about to push, THEN pushes by digest. The bytes
	# that were validated are the bytes that ship.
	# manifest packages: write join the per-arch digests into :<version>-<lane>
	# attest id-token: write provenance over the manifest digest
	# promote packages: write move :latest-<lane> only after everything is green
	#
	# Pull requests and manual dispatch stop after verify.

	on:
	push:
	tags: ['v*']
	# main publishes moving :main-<lane> images. Only container INFRASTRUCTURE
	# paths trigger it: main takes dozens of pushes a day and three lanes on two
	# architectures each time is prohibitive, so product changes ride the
	# nightly below and container changes rebuild immediately.
	branches: [main]
	paths:
	- 'docker/**'
	- 'release/container-matrix.json'
	- 'scripts/validate-container-image.py'
	- 'scripts/container_tags.py'
	- 'scripts/build-cpu-release.sh'
	- 'scripts/build-linux-accelerator-release.sh'
	- '.github/workflows/containers.yml'
	schedule:
	# Nightly, so a main image is never more than a day behind the tree.
	- cron: '0 4 * * *'
	pull_request:
	workflow_dispatch:
	inputs:
	full_matrix:
	description: >-
	Build every lane on both architectures, including the ten-SM fat cuda
	lane that pull requests skip for cost. Use this to prove a lane that
	is not in the pull-request matrix.
	type: boolean
	default: false

	permissions:
	contents: read

	env:
	REGISTRY_PACKAGE: ghcr.io/mudler/vllm.cpp

	# 2026-08-15 (#822): this workflow had NO concurrency policy at all and was
	# named as the largest source of superseded queued runs -- 32 obsolete runs were
	# cancelled by hand on 2026-08-14.
	#
	# Pull request and main-push runs are latest-only. A TAG run keys on the tag and
	# is never cancelled: publish, manifest, attest and promote push by digest, and
	# a cancelled publish can leave a manifest half-joined. Schedule and dispatch
	# keep their own partition through the event_name token.
	concurrency:
	group: containers-${{ github.event_name }}-${{ github.event.pull_request.number \|\| github.ref }}-${{ github.repository }}
	cancel-in-progress: ${{ (github.event_name == 'pull_request' \|\| github.event_name == 'push') && !startsWith(github.ref, 'refs/tags/') }}

	jobs:
	plan:
	permissions:
	contents: read
	runs-on: ubuntu-latest
	outputs:
	version: ${{ steps.plan.outputs.version }}
	is_release: ${{ steps.plan.outputs.is_release }}
	is_main: ${{ steps.plan.outputs.is_main }}
	publishes: ${{ steps.plan.outputs.publishes }}
	verify_matrix: ${{ steps.matrix.outputs.verify }}
	publish_matrix: ${{ steps.matrix.outputs.publish }}
	steps:
	- uses: actions/checkout@v4
	- name: Resolve the version and validate it against the tag
	id: plan
	run: \|
	set -euo pipefail
	version=$(sed -n 's/^project(vllm_cpp VERSION $[^ ]$./\1/p' CMakeLists.txt)
	test -n "$version"
	if [ "${GITHUB_REF_TYPE}" = "tag" ]; then
	# The tag is untrusted input until it matches the tree.
	test "${GITHUB_REF_NAME}" = "v${version}"
	echo "is_release=true" >> "$GITHUB_OUTPUT"
	else
	echo "is_release=false" >> "$GITHUB_OUTPUT"
	fi
	# A main publish is a push to main or the nightly -- never a pull
	# request, which has no credentials and must not reach a registry.
	is_main=false
	if [ "${GITHUB_REF_TYPE}" != "tag" ] \
	&& [ "${GITHUB_REF_NAME}" = "main" ] \
	&& [ "${GITHUB_EVENT_NAME}" != "pull_request" ]; then
	is_main=true
	fi
	echo "is_main=${is_main}" >> "$GITHUB_OUTPUT"
	if [ "${is_main}" = "true" ] \|\| [ "${GITHUB_REF_TYPE}" = "tag" ]; then
	echo "publishes=true" >> "$GITHUB_OUTPUT"
	else
	echo "publishes=false" >> "$GITHUB_OUTPUT"
	fi
	echo "version=${version}" >> "$GITHUB_OUTPUT"
	- name: Gate the container matrix against the Dockerfile
	run: python3 scripts/check-container-matrix.py
	- name: Resolve the lane x architecture build matrices
	id: matrix
	run: \|
	set -euo pipefail
	# A release run builds every lane on both architectures. A pull
	# request builds the reduced set the matrix opts in, because a ten-SM
	# fat CUDA image does not fit a hosted runner budget per push. Nothing
	# is published without the full set: publish rebuilds and revalidates
	# each lane immediately before pushing it.
	if [ "${{ steps.plan.outputs.is_release }}" = "true" ] \
	\|\| [ "${{ inputs.full_matrix }}" = "true" ]; then
	echo "verify=$(python3 scripts/container_tags.py --build-matrix --release)" >> "$GITHUB_OUTPUT"
	else
	echo "verify=$(python3 scripts/container_tags.py --build-matrix)" >> "$GITHUB_OUTPUT"
	fi
	echo "publish=$(python3 scripts/container_tags.py --build-matrix --release)" >> "$GITHUB_OUTPUT"
	- name: Gate this workflow against its own contract
	run: python3 scripts/check-container-workflow.py
	- name: Builder stages can build what the runtime stage advertises
	# Issue #1517. The runtime stage carried libssl3 while no builder
	# stage carried libssl-dev, so every image this workflow published
	# shipped a server that could not fetch from huggingface.co, and
	# every other gate here stayed green.
	run: python3 scripts/check-build-runtime-deps.py

	verify:
	needs: plan
	permissions:
	contents: read
	strategy:
	fail-fast: false
	matrix:
	include: ${{ fromJSON(needs.plan.outputs.verify_matrix) }}
	runs-on: ${{ matrix.runner }}
	# Issue #1548. No job here declared timeout-minutes, so the default six
	# hours applied and a job that died at 35 minutes reported only "the runner
	# has received a shutdown signal" and exit 143. That message is the same
	# whether the cause was memory, a reclaimed VM, or a genuine hang, and the
	# absence of a budget is what made the three indistinguishable.
	#
	# The value is deliberately LOOSE. Its purpose is to LABEL the next failure,
	# not to police the duration: a job that dies before the budget is external,
	# and a job that dies at the budget hung. The slowest lane is cuda, which
	# the 512-of-787 data point projects near two hours at the cap the build
	# step sets, so 300
	# minutes leaves room for the arm64 legs this repository has never built
	# while still bounding the job below the default.
	timeout-minutes: 300
	steps:
	- uses: actions/checkout@v4
	- uses: docker/setup-buildx-action@v3
	- name: Build the lane image for this architecture
	run: \|
	set -euo pipefail
	# Build parallelism is LANE-AWARE, and issue #1548 is why.
	#
	# Every .cu file in the cuda lane is compiled for the ten device
	# architectures scripts/build-linux-accelerator-release.sh:24 sets, so
	# one compiler process there holds many times the resident set of a
	# cpu or vulkan translation unit, and $(nproc) of them run at once.
	# Run 32447481128 died at object 512 of 787 about 35 minutes in with
	# exit 143 and "the runner has received a shutdown signal".
	#
	# That reading is a HYPOTHESIS. Memory exhaustion and GitHub
	# infrastructure reclamation produce the same message and the same
	# exit code, and the available logs cannot separate them. The cap
	# below removes the one of the two this repository can control.
	#
	# 2 is measured, not guessed. ci.yml:801 builds the SAME ten-SM fat
	# gencode set on a hosted runner at --parallel 2 and is green. The
	# 512-of-787 data point puts a halved build near two hours, far
	# inside the timeout below, which answers the concern in
	# .agents/specs/container-images.md:200-204 that two jobs would not
	# finish. The cpu and vulkan lanes are not failing and keep $(nproc).
	jobs=$(nproc)
	if [ "${{ matrix.lane }}" = "cuda" ]; then jobs=2; fi
	docker buildx build \
	--file docker/Dockerfile \
	--target "${{ matrix.lane }}" \
	--platform "${{ matrix.platform }}" \
	--build-arg "VERSION=${{ needs.plan.outputs.version }}" \
	--build-arg "SOURCE_SHA=${GITHUB_SHA}" \
	--build-arg "SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD)" \
	--build-arg "EVIDENCE_URL=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
	--build-arg "JOBS=${jobs}" \
	--tag "vllm-cpp-verify:${{ matrix.lane }}" \
	--load \
	.
	- name: Validate the built image
	run: \|
	python3 scripts/validate-container-image.py \
	--image "vllm-cpp-verify:${{ matrix.lane }}" \
	--lane "${{ matrix.lane }}" \
	--version "${{ needs.plan.outputs.version }}" \
	--expect-revision "${GITHUB_SHA}"

	publish:
	needs: [plan, verify]
	if: needs.plan.outputs.publishes == 'true'
	permissions:
	contents: read
	packages: write
	strategy:
	fail-fast: false
	matrix:
	include: ${{ fromJSON(needs.plan.outputs.publish_matrix) }}
	runs-on: ${{ matrix.runner }}
	# Issue #1548. No job here declared timeout-minutes, so the default six
	# hours applied and a job that died at 35 minutes reported only "the runner
	# has received a shutdown signal" and exit 143. That message is the same
	# whether the cause was memory, a reclaimed VM, or a genuine hang, and the
	# absence of a budget is what made the three indistinguishable.
	#
	# The value is deliberately LOOSE. Its purpose is to LABEL the next failure,
	# not to police the duration: a job that dies before the budget is external,
	# and a job that dies at the budget hung. The slowest lane is cuda, which
	# the 512-of-787 data point projects near two hours at the cap the build
	# step sets, so 300
	# minutes leaves room for the arm64 legs this repository has never built
	# while still bounding the job below the default.
	timeout-minutes: 300
	steps:
	- uses: actions/checkout@v4
	- uses: docker/setup-buildx-action@v3
	- uses: docker/login-action@v3
	with:
	registry: ghcr.io
	username: ${{ github.actor }}
	password: ${{ secrets.GITHUB_TOKEN }}
	- name: Refuse to overwrite an existing immutable tag
	# Version tags only. :main-<lane> moves by design and is skipped here;
	# what must never happen is a republished :<version>-<lane>.
	if: needs.plan.outputs.is_release == 'true'
	run: \|
	set -euo pipefail
	tag="${REGISTRY_PACKAGE}:${{ needs.plan.outputs.version }}-${{ matrix.lane }}"
	if docker buildx imagetools inspect "$tag" >/dev/null 2>&1; then
	echo "::error::${tag} already exists; version tags are immutable" >&2
	exit 1
	fi
	- name: Build the exact image this job will push
	run: \|
	set -euo pipefail
	# The same lane-aware cap the verify job takes, and issue #1548
	# records the reason in full there. This job rebuilds the identical
	# image, so the two must never resolve a different JOBS: a cap that
	# held in verify and not here would let publish die on exactly the
	# build verify proved.
	jobs=$(nproc)
	if [ "${{ matrix.lane }}" = "cuda" ]; then jobs=2; fi
	docker buildx build \
	--file docker/Dockerfile \
	--target "${{ matrix.lane }}" \
	--platform "${{ matrix.platform }}" \
	--build-arg "VERSION=${{ needs.plan.outputs.version }}" \
	--build-arg "SOURCE_SHA=${GITHUB_SHA}" \
	--build-arg "SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD)" \
	--build-arg "EVIDENCE_URL=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
	--build-arg "JOBS=${jobs}" \
	--tag "vllm-cpp-publish:${{ matrix.lane }}" \
	--load \
	.
	- name: Validate the image immediately before pushing it
	run: \|
	python3 scripts/validate-container-image.py \
	--image "vllm-cpp-publish:${{ matrix.lane }}" \
	--lane "${{ matrix.lane }}" \
	--version "${{ needs.plan.outputs.version }}" \
	--expect-revision "${GITHUB_SHA}"
	- name: Push by digest
	id: push
	run: \|
	set -euo pipefail
	docker tag "vllm-cpp-publish:${{ matrix.lane }}" "${REGISTRY_PACKAGE}:stage"
	docker push "${REGISTRY_PACKAGE}:stage"
	# `docker push --quiet` prints the image REFERENCE, not the digest, so
	# read back what the registry actually stored.
	digest=$(docker image inspect "${REGISTRY_PACKAGE}:stage" \
	--format '{{index .RepoDigests 0}}' \| sed 's/.*@//')
	case "$digest" in sha256:) ;; ) echo "::error::no digest" >&2; exit 1 ;; esac
	echo "digest=${digest}" >> "$GITHUB_OUTPUT"
	- name: Record the exact digest for the manifest stage
	run: \|
	mkdir -p digests
	arch=$(echo "${{ matrix.platform }}" \| tr '/' '-')
	echo "${{ steps.push.outputs.digest }}" > "digests/${{ matrix.lane }}-${arch}"
	- uses: actions/upload-artifact@v4
	with:
	name: container-digest-${{ matrix.lane }}-${{ strategy.job-index }}
	path: digests/*
	if-no-files-found: error
	retention-days: 7

	manifest:
	needs: [plan, publish]
	if: needs.plan.outputs.publishes == 'true'
	permissions:
	contents: read
	packages: write
	runs-on: ubuntu-latest
	outputs:
	digests: ${{ steps.create.outputs.digests }}
	steps:
	- uses: actions/checkout@v4
	- uses: actions/download-artifact@v4
	with:
	pattern: container-digest-*
	path: digests
	merge-multiple: true
	- uses: docker/login-action@v3
	with:
	registry: ghcr.io
	username: ${{ github.actor }}
	password: ${{ secrets.GITHUB_TOKEN }}
	- name: Join the per-architecture digests into one manifest per lane
	id: create
	run: \|
	set -euo pipefail
	version="${{ needs.plan.outputs.version }}"
	for lane in cpu vulkan cuda; do
	refs=""
	for file in digests/${lane}-linux-*; do
	refs="${refs} ${REGISTRY_PACKAGE}@$(cat "$file")"
	done
	test -n "${refs}"
	if [ "${{ needs.plan.outputs.is_release }}" = "true" ]; then
	tag="${REGISTRY_PACKAGE}:${version}-${lane}"
	else
	# A main image is a moving convenience tag, never a version.
	tag="${REGISTRY_PACKAGE}:main-${lane}"
	fi
	docker buildx imagetools create --tag "${tag}" ${refs}
	done

	attest:
	needs: [plan, manifest]
	if: needs.plan.outputs.publishes == 'true'
	permissions:
	contents: read
	id-token: write
	attestations: write
	packages: read
	runs-on: ubuntu-latest
	steps:
	- name: Resolve the manifest digest for each lane
	id: resolve
	run: \|
	set -euo pipefail
	version="${{ needs.plan.outputs.version }}"
	if [ "${{ needs.plan.outputs.is_release }}" = "true" ]; then prefix="${version}-"; else prefix="main-"; fi
	for lane in cpu vulkan cuda; do
	digest=$(docker buildx imagetools inspect \
	"${REGISTRY_PACKAGE}:${prefix}${lane}" --format '{{json .Manifest.Digest}}' \| tr -d '"')
	echo "${lane}=${digest}" >> "$GITHUB_OUTPUT"
	done
	- uses: actions/attest-build-provenance@v2
	with:
	subject-name: ${{ env.REGISTRY_PACKAGE }}
	subject-digest: ${{ steps.resolve.outputs.cpu }}
	push-to-registry: true
	- uses: actions/attest-build-provenance@v2
	with:
	subject-name: ${{ env.REGISTRY_PACKAGE }}
	subject-digest: ${{ steps.resolve.outputs.vulkan }}
	push-to-registry: true
	- uses: actions/attest-build-provenance@v2
	with:
	subject-name: ${{ env.REGISTRY_PACKAGE }}
	subject-digest: ${{ steps.resolve.outputs.cuda }}
	push-to-registry: true

	promote:
	needs: [plan, attest]
	if: needs.plan.outputs.is_release == 'true'
	permissions:
	contents: read
	packages: write
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: docker/login-action@v3
	with:
	registry: ghcr.io
	username: ${{ github.actor }}
	password: ${{ secrets.GITHUB_TOKEN }}
	- name: Move the lane pointers, and :latest to the default lane
	run: \|
	set -euo pipefail
	python3 scripts/container_tags.py \
	--version "${{ needs.plan.outputs.version }}" --moving > moving-tags.txt
	test -s moving-tags.txt
	while read -r source target; do
	docker buildx imagetools create --tag "$target" "$source"
	done < moving-tags.txt

Navigation Menu

fix(LTX25-PHASE-RESIDUE): 92% of the phase table's un-named time was one region, and two wall-clock ratios were measuring it (#1536) #1328

Workflow file

fix(LTX25-PHASE-RESIDUE): 92% of the phase table's un-named time was one region, and two wall-clock ratios were measuring it (#1536) #1328

Uh oh!

Workflow file for this run