Repository navigation
350 lines (324 loc) · 16.5 KB
/
Copy pathexamples-compile.yml
File metadata and controls
350 lines (324 loc) · 16.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
---
name: examples-compile
# Path-filtered SIMT lane (same roots as ci.yml).
on:
# Self-hosted runners reject pull_request. copy-pr-bot mirrors trusted PRs
# to pull-request/<n>, and this workflow triggers on that push.
push:
# No workflow-level paths: filter; the changes job below gates the jobs.
branches:
- main
- "pull-request/[0-9]+"
# Each commit on main gets its own group, so it starts at once. Any other ref
# still shares one group and cancels the older run.
concurrency:
group: ${{ github.workflow }}-${{ github.ref_name }}${{ github.ref_name == 'main' && format('-{0}', github.sha) || '' }}
cancel-in-progress: ${{ github.ref_name != 'main' }}
permissions:
contents: read
# A container job's default shell is dash. The LLVM install and the
# smoketest steps use `set -o pipefail`.
defaults:
run:
shell: bash
jobs:
# Path gate. copy-pr-bot creates pull-request/<n> at an existing SHA, and
# GitHub often treats that push as having no path changes, so a workflow-level
# `paths:` filter would skip the whole run. This job diffs against main
# instead, as cutile-rs.yml does, and every job below runs when no file diff
# is detected. Other events (schedule, workflow_dispatch) skip the gate.
changes:
if: github.event_name == 'push'
runs-on: linux-amd64-cpu4
timeout-minutes: 5
permissions:
contents: read
outputs:
relevant: ${{ steps.changes.outputs.relevant }}
any_changed: ${{ steps.changes.outputs.any_changed }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: dorny/paths-filter@0e4a8c6effa4802afeda77dc8d303f8176d7dfad # v3.0.4
id: changes
with:
filters: |
relevant:
- 'cuda-oxide/**'
- 'cuda-bindings/**'
- 'cuda-core/**'
- 'cuda-core-derive/**'
- 'cuda-repr/**'
- 'cuda-async/**'
- 'Cargo.toml'
- 'Cargo.lock'
- '.github/workflows/**'
- '.github/actions/setup-cuda-container/**'
# `*` covers a root file, `**` every nested path. Both false means
# no file diff was detected.
any_changed:
- '*'
- '**'
examples-compile:
needs: changes
if: ${{ !cancelled() && (github.event_name != 'push' || needs.changes.outputs.relevant == 'true' || needs.changes.outputs.any_changed != 'true') }}
name: compile all examples (device pipeline, GPU-less)${{ matrix.cuda.suffix }}
runs-on: linux-amd64-cpu4
# CUDA devel image (see the matrix): nvcc, cudart, libcurand, and
# libnvJitLink. The materialization gate below needs nvJitLink, which the
# devel image ships. rustup is installed in the container.
container:
image: ${{ matrix.cuda.image }}
env:
DEBIAN_FRONTEND: noninteractive
env:
CUDA_TOOLKIT_PATH: /usr/local/cuda
# Every example crate: backend build + shared dep tree + per-example
# device codegen. Typically well under an hour with the shared
# CARGO_TARGET_DIR; the timeout leaves headroom for cold runners.
# Leave job-level headroom around the 50-minute full compile and the
# focused 10-minute materialization gate, plus toolkit/LLVM setup.
timeout-minutes: 75
strategy:
fail-fast: false
# The baseline lane keeps its existing job name (empty suffix); the newer
# image is a separate lane. The nvcr.io image has the same digest as Docker
# Hub's nvidia/cuda:13.5.0-devel-ubuntu24.04 and no anonymous pull limit.
matrix:
cuda:
- image: nvidia/cuda:13.3.1-devel-ubuntu24.04
suffix: ""
- image: nvcr.io/nvidia/cuda:13.5.0-devel-ubuntu24.04
suffix: " (CUDA 13.5.0)"
steps:
# $/ installs git before checkout. The CUDA image has no git; without
# it checkout stores a tarball. The same step installs python3 for
# sync-example-locks.sh.
- uses: $/.github/actions/setup-cuda-container
- name: Clone repository
uses: actions/checkout@v4
# Every example workspace must pin the same dependency versions. The
# shared CARGO_TARGET_DIR below keys build caches on the resolved
# dependency subtree, so one drifted Cargo.lock silently costs a full
# dependency-tree rebuild (bindgen over cuda.h included) for that one
# example -- a 15-70s spike in an otherwise ~2s-per-example run.
# Fails fast, before LLVM and the compile.
- name: Example lockfiles resolve identical dependency versions
run: bash cuda-oxide/scripts/sync-example-locks.sh --check
# The device pipeline shells out to LLVM's llc/opt for PTX
# generation. This pin is the FLOOR GATE: if someone lands IR that
# needs a newer llc, this job is where the floor bump becomes
# visible instead of on contributors' machines. LLVM 23 is the true
# floor: the bf16x2 to packed-FP8 conversions (PTX 9.1 forms, used
# by generated_intrinsics_blackwell) need `+ptx91`, which llc-22
# does not know. It ignored the unknown feature and emitted
# `.version 8.8` PTX that failed only at JIT; the pipeline now
# refuses a PTX 9.1 request below LLVM 23 with a named error, so an
# under-floor llc fails loudly here. The previous floor, LLVM 22 for
# the ldmatrix/stmatrix family (llc-21 leaked unknown intrinsics
# into the PTX as `.extern .func`), is covered by the same gate.
# The container is Ubuntu 24.04, so VERSION_CODENAME is noble.
# If apt.llvm.org drops llvm-toolchain-noble-23, this update fails
# loudly; recover by bumping the LLVM major here.
- name: Install LLVM 23 (llc/opt for the NVPTX backend)
run: |
set -euo pipefail
source /etc/os-release
install -d -m 0755 /etc/apt/keyrings
wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key \
| tee /etc/apt/keyrings/apt-llvm-org.asc >/dev/null
echo "deb [signed-by=/etc/apt/keyrings/apt-llvm-org.asc] https://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-23 main" \
| tee /etc/apt/sources.list.d/llvm-23.list
apt-get update
apt-get install -y --no-install-recommends llvm-23
llc-23 --version | head -2
opt-23 --version | head -2
- name: Compile every example (smoketest --compile-only)
# Step-level timeout, below the job's 75: a job-level timeout
# CANCELS the job and `if: failure()` never fires, losing the
# logs. A step timeout fails the step, so the upload still runs.
timeout-minutes: 50
env:
CUDA_OXIDE_LLC: /usr/bin/llc-23
# Each example is its own cargo workspace; without a shared
# target dir the dep tree rebuilds ~85 times and blows the
# runner's ~14 GB disk (same trick as clippy.yml). smoketest.sh
# honours an externally-set CARGO_TARGET_DIR and still builds
# the backend .so at its fixed discovery path.
CARGO_TARGET_DIR: ${{ runner.temp }}/examples-shared-target
SMOKETEST_LOG_DIR: ${{ runner.temp }}/smoketest-logs
# The runner's ~14 GB disk also holds the CUDA toolkit, LLVM,
# cargo-oxide, and ~200 example workspaces; rustc incremental dirs
# under rustc-codegen-cuda/target/ are what used to fill it and fail
# later examples with ENOSPC / ld bus errors. The cutile-rs source
# tree itself is a few MB and must stay: cutile_inter_kernel
# path-depends on cutile-rs/cutile and cutile-rs/cutile-compiler.
CARGO_INCREMENTAL: "0"
run: |
set -euo pipefail
sysroot="$(rustc --print sysroot)"
export LIBRARY_PATH="${sysroot}/lib:${LIBRARY_PATH:-}"
export LD_LIBRARY_PATH="${sysroot}/lib:${LD_LIBRARY_PATH:-}"
cuda-oxide/scripts/smoketest.sh --compile-only --no-color
# Builds shared_static_names, then copies with added kernels, in
# release and full-debug, and requires the original kernels' PTX to be
# unchanged (#1477). Reuses the shared target dir, so it adds only the
# example's own device codegen.
- name: Adding a kernel keeps other kernels' PTX unchanged
timeout-minutes: 10
working-directory: cuda-oxide
env:
CUDA_OXIDE_LLC: /usr/bin/llc-23
CARGO_TARGET_DIR: ${{ runner.temp }}/examples-shared-target
CARGO_INCREMENTAL: "0"
run: |
set -euo pipefail
sysroot="$(rustc --print sysroot)"
export LIBRARY_PATH="${sysroot}/lib:${LIBRARY_PATH:-}"
export LD_LIBRARY_PATH="${sysroot}/lib:${LD_LIBRARY_PATH:-}"
bash scripts/check-shared-static-names.sh
- name: Artifact modes keep shared Cargo dependencies fresh
timeout-minutes: 10
working-directory: cuda-oxide
env:
CUDA_OXIDE_LLC: /usr/bin/llc-23
CARGO_TARGET_DIR: ${{ runner.temp }}/examples-shared-target
CARGO_INCREMENTAL: "0"
run: |
set -euo pipefail
sysroot="$(rustc --print sysroot)"
export LIBRARY_PATH="${sysroot}/lib:${LIBRARY_PATH:-}"
export LD_LIBRARY_PATH="${CUDA_TOOLKIT_PATH}/lib64:${sysroot}/lib:${LD_LIBRARY_PATH:-}"
assert_embedded_artifact() {
local binary="$1"
local expected_version="$2"
local expected_kind="$3"
local expected_target="$4"
local expected_strings=("${@:5}")
local section_file="${RUNNER_TEMP}/$(basename "${binary}")-${expected_kind}.oxart"
objcopy --dump-section ".oxart=${section_file}" "${binary}"
local version name_len target_len target record_offset kind data_offset data_len magic
version="$(od -An -tu2 -j8 -N2 "${section_file}" | tr -d '[:space:]')"
name_len="$(od -An -tu2 -j16 -N2 "${section_file}" | tr -d '[:space:]')"
target_len="$(od -An -tu2 -j18 -N2 "${section_file}" | tr -d '[:space:]')"
target="$(od -An -tc -j$((32 + name_len)) -N"${target_len}" "${section_file}" | tr -d '[:space:]')"
record_offset=$((32 + name_len + target_len))
kind="$(od -An -tu2 -j"${record_offset}" -N2 "${section_file}" | tr -d '[:space:]')"
data_offset="$(od -An -tu4 -j$((record_offset + 4)) -N4 "${section_file}" | tr -d '[:space:]')"
data_len="$(od -An -tu4 -j$((record_offset + 8)) -N4 "${section_file}" | tr -d '[:space:]')"
test "${version}" = "${expected_version}"
test "${target}" = "${expected_target}"
test "${kind}" = "${expected_kind}"
if [ "${expected_kind}" = "512" ]; then
magic="$(od -An -tx1 -j"${data_offset}" -N4 "${section_file}" | tr -d '[:space:]')"
test "${magic}" = "7f454c46"
if [ "${#expected_strings[@]}" -gt 0 ]; then
local payload_file="${section_file}.cubin"
dd if="${section_file}" of="${payload_file}" bs=1 \
skip="${data_offset}" count="${data_len}" status=none
local expected_string
for expected_string in "${expected_strings[@]}"; do
grep -Fqx "${expected_string}" < <(strings "${payload_file}")
done
fi
fi
}
assert_only_expected_rebuilt() {
local log="$1"
local allowed="$2"
shift 2
local unexpected
unexpected="$(
sed -nE 's/^[[:space:]]*Compiling ([^[:space:]]+).*/\1/p' "${log}" \
| sort -u \
| grep -Ev "${allowed}" \
|| true
)"
if [ -n "${unexpected}" ]; then
echo "Codegen mode change rebuilt shared dependencies:" >&2
echo "${unexpected}" >&2
return 1
fi
local package
for package in "$@"; do
if ! grep -Eq "^[[:space:]]*Compiling ${package}([[:space:]]|$)" "${log}"; then
echo "Expected ${package} to rebuild, but it stayed fresh" >&2
return 1
fi
done
}
cargo oxide build vecadd --arch sm_86 2>&1 \
| tee "${RUNNER_TEMP}/vecadd-ptx-build.log"
assert_only_expected_rebuilt \
"${RUNNER_TEMP}/vecadd-ptx-build.log" '^vecadd$' vecadd
assert_embedded_artifact "${CARGO_TARGET_DIR}/release/vecadd" 1 256 sm_86
cargo oxide build vecadd --materialize-cubin --arch sm_86 2>&1 \
| tee "${RUNNER_TEMP}/vecadd-cubin-build.log"
assert_only_expected_rebuilt \
"${RUNNER_TEMP}/vecadd-cubin-build.log" '^vecadd$' vecadd
# Default compile policy remains wire-compatible artifact v1; the
# payload kind is what must change from PTX (256) to cubin (512).
assert_embedded_artifact "${CARGO_TARGET_DIR}/release/vecadd" 1 512 sm_86
# IKET metadata is consumed by the profiler rather than device code.
# Guard the complete libNVVM + nvJitLink path: an IR-only assertion
# would miss internalization dropping these globals from the cubin.
cargo oxide build iket_trace --materialize-cubin --arch sm_120 2>&1 \
| tee "${RUNNER_TEMP}/iket-trace-cubin-build.log"
assert_only_expected_rebuilt \
"${RUNNER_TEMP}/iket-trace-cubin-build.log" '^iket_trace$' iket_trace
assert_embedded_artifact \
"${CARGO_TARGET_DIR}/release/iket_trace" 1 512 sm_120 \
__iket_meta_info \
vecadd.completed.with.a.name.longer.than.thirty.one.bytes
# Switching back must invalidate vecadd's device artifact without
# throwing away the shared host dependency cache.
cargo oxide build vecadd --arch sm_86 2>&1 \
| tee "${RUNNER_TEMP}/vecadd-ptx-return-build.log"
assert_only_expected_rebuilt \
"${RUNNER_TEMP}/vecadd-ptx-return-build.log" '^vecadd$' vecadd
assert_embedded_artifact "${CARGO_TARGET_DIR}/release/vecadd" 1 256 sm_86
# A kernel-producing path dependency must be invalidated too, while
# ordinary CUDA/host dependencies remain reusable.
cargo oxide build cross_crate_kernel --arch sm_86 2>&1 \
| tee "${RUNNER_TEMP}/cross-crate-sm86-build.log"
assert_only_expected_rebuilt \
"${RUNNER_TEMP}/cross-crate-sm86-build.log" \
'^(kernel-lib|cross_crate_kernel)$' kernel-lib cross_crate_kernel
cargo oxide build cross_crate_kernel --arch sm_90 2>&1 \
| tee "${RUNNER_TEMP}/cross-crate-sm90-build.log"
assert_only_expected_rebuilt \
"${RUNNER_TEMP}/cross-crate-sm90-build.log" \
'^(kernel-lib|cross_crate_kernel)$' kernel-lib cross_crate_kernel
# Scalar math has a Turing floor. Build every form at sm_75 through
# both LLVM (PTX) and libNVVM + nvJitLink (cubin), so a raised floor
# fails here instead of in a downstream sm_75 build.
cargo oxide build scalar_math_sm75 --arch sm_75 2>&1 \
| tee "${RUNNER_TEMP}/scalar-math-sm75-ptx-build.log"
assert_embedded_artifact \
"${CARGO_TARGET_DIR}/release/scalar_math_sm75" 1 256 sm_75
cargo oxide build scalar_math_sm75 --materialize-cubin --arch sm_75 2>&1 \
| tee "${RUNNER_TEMP}/scalar-math-sm75-cubin-build.log"
assert_embedded_artifact \
"${CARGO_TARGET_DIR}/release/scalar_math_sm75" 1 512 sm_75
- name: Upload failure logs
if: failure()
uses: actions/upload-artifact@v4
with:
# Artifact names are unique per run; the baseline lane keeps its name
# (empty suffix) and the newer lane adds its suffix.
name: smoketest-compile-logs${{ matrix.cuda.suffix }}
# One glob per log the artifact-modes step writes. `iket-trace-*`
# matched neither of the other two, so the one assertion that
# exercises the full libNVVM + nvJitLink path -- and the only one
# that can fail on internalization dropping `__iket_meta_info` from
# the cubin -- was the one whose log never reached the artifact.
# `if-no-files-found: ignore` below made that silent.
path: |
${{ runner.temp }}/smoketest-logs/
${{ runner.temp }}/vecadd-*-build.log
${{ runner.temp }}/iket-trace-*-build.log
${{ runner.temp }}/cross-crate-*-build.log
${{ runner.temp }}/scalar-math-sm75-*-build.log
if-no-files-found: ignore
retention-days: 7