From 390c2f626a81a8601d3877848e2a671445b4548b Mon Sep 17 00:00:00 2001 From: sametd Date: Wed, 15 Apr 2026 19:46:43 +0200 Subject: [PATCH 1/7] ci: add Docker-based CI image with multi-stage build Pre-bakes Rust 1.94.0 + clippy/rustfmt + wasm-pack 0.14.0 + uv on Ubuntu 24.04. Python and Node.js are installed at CI runtime to keep the image at 1.55 GB. - Multi-stage build: wasm-pack compiled in builder, only binary copied - libclang1-18 instead of libclang-18-dev (saves ~250 MB) - Static .a libs stripped, shared libs stripped of debug symbols - All tool versions pinned as ARGs at the top for easy bumps - VERSION file for semver tagging (1.0.0) --- .github/ci/Dockerfile | 86 +++++++++++++++++++++++++++++++++++++++++++ .github/ci/VERSION | 1 + 2 files changed, 87 insertions(+) create mode 100644 .github/ci/Dockerfile create mode 100644 .github/ci/VERSION diff --git a/.github/ci/Dockerfile b/.github/ci/Dockerfile new file mode 100644 index 00000000..d928b47b --- /dev/null +++ b/.github/ci/Dockerfile @@ -0,0 +1,86 @@ +# ──────────────────────────────────────────────────────────────── +# tensogram CI image +# +# Pre-bakes Rust toolchain + system libs. Python and Node.js are +# installed at CI runtime by uv / curl (adds ~10 s per job that +# needs them, but shrinks the image by ~430 MB). +# +# docker build -t tensogram-ci:test .github/ci/ +# ──────────────────────────────────────────────────────────────── + +# ── Pinned versions (edit here to bump) ───────────────────────── +ARG UV_VERSION=0.11.6 + +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv + +FROM ubuntu:24.04 AS builder + +ARG RUST_VERSION=1.94.0 +ARG WASM_PACK_VERSION=0.14.0 + +ENV DEBIAN_FRONTEND=noninteractive \ + LANG=C.UTF-8 \ + RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH=/usr/local/cargo/bin:$PATH + +RUN set -eux \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential pkg-config curl ca-certificates \ + && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ + | sh -s -- -y --default-toolchain "${RUST_VERSION}" \ + --profile minimal -c clippy -c rustfmt \ + && rustup target add wasm32-unknown-unknown \ + && cargo install wasm-pack --version "${WASM_PACK_VERSION}" --locked \ + && rm -rf "${CARGO_HOME}"/registry "${CARGO_HOME}"/git /tmp/* \ + "${RUSTUP_HOME}"/toolchains/*/share/doc \ + "${RUSTUP_HOME}"/toolchains/*/share/man + +# ── Final image ───────────────────────────────────────────────── +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive \ + LANG=C.UTF-8 \ + RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + LIBCLANG_PATH=/usr/lib/x86_64-linux-gnu \ + PATH=/usr/local/cargo/bin:$PATH + +# ── System packages ───────────────────────────────────────────── +# Ubuntu 24.04 ships cmake 3.28 and LLVM 18 — zero PPAs needed. +RUN set -eux \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential pkg-config git curl ca-certificates \ + cmake \ + libclang1-18 libclang-common-18-dev \ + libnetcdf-dev libhdf5-dev libeccodes-dev \ + && rm -f /usr/lib/x86_64-linux-gnu/libicu*.a \ + /usr/lib/x86_64-linux-gnu/libhdf5*.a \ + /usr/lib/x86_64-linux-gnu/libssl.a \ + /usr/lib/x86_64-linux-gnu/libcrypto.a \ + /usr/lib/x86_64-linux-gnu/libxml2.a \ + /usr/lib/x86_64-linux-gnu/libcurl*.a \ + && rm -rf /usr/lib/llvm-18/include \ + && strip --strip-unneeded /usr/lib/x86_64-linux-gnu/lib*.so* 2>/dev/null || true \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ + /usr/share/doc /usr/share/man /usr/share/info \ + /usr/share/lintian /usr/share/cmake-3.28/Help + +# ── Rust toolchain + wasm-pack (from builder) ────────────────── +COPY --from=builder /usr/local/rustup /usr/local/rustup +COPY --from=builder /usr/local/cargo /usr/local/cargo + +# ── uv (Python + Node installed at CI runtime) ───────────────── +COPY --from=uv /uv /uvx /usr/local/bin/ + +# ── Smoke test ────────────────────────────────────────────────── +RUN set -eux \ + && rustc --version && cargo --version \ + && cmake --version \ + && wasm-pack --version \ + && uv --version + +WORKDIR /workspace diff --git a/.github/ci/VERSION b/.github/ci/VERSION new file mode 100644 index 00000000..3eefcb9d --- /dev/null +++ b/.github/ci/VERSION @@ -0,0 +1 @@ +1.0.0 From 00ce28bec2f3d52fb8bd64380fa73942f8f624af Mon Sep 17 00:00:00 2001 From: sametd Date: Wed, 15 Apr 2026 19:46:49 +0200 Subject: [PATCH 2/7] ci: add workflow to build and push CI image to ECCR Triggers on pushes to main touching .github/ci/** and on PRs (build-only, no push). Publishes four tags per build: :1.0.0, :1.0, :1, :latest. Uses docker/build-push-action with GHA cache. --- .github/workflows/ci-image.yml | 59 ++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 .github/workflows/ci-image.yml diff --git a/.github/workflows/ci-image.yml b/.github/workflows/ci-image.yml new file mode 100644 index 00000000..c522504d --- /dev/null +++ b/.github/workflows/ci-image.yml @@ -0,0 +1,59 @@ +name: CI Image + +on: + push: + branches: [main] + paths: [".github/ci/**"] + pull_request: + paths: [".github/ci/**"] + workflow_dispatch: + +concurrency: + group: ci-image-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + IMAGE: eccr.ecmwf.int/tensogram/ci + +jobs: + build-and-push: + name: Build CI image + runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04] + steps: + - uses: actions/checkout@v4 + + - name: Read version + id: version + run: | + VERSION=$(cat .github/ci/VERSION | tr -d '[:space:]') + MAJOR=$(echo "$VERSION" | cut -d. -f1) + MINOR=$(echo "$VERSION" | cut -d. -f1-2) + echo "full=$VERSION" >> "$GITHUB_OUTPUT" + echo "major=$MAJOR" >> "$GITHUB_OUTPUT" + echo "minor=$MINOR" >> "$GITHUB_OUTPUT" + + - uses: docker/setup-buildx-action@v3 + + - name: Login to ECCR + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: eccr.ecmwf.int + username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} + password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: .github/ci/ + push: ${{ github.event_name != 'pull_request' }} + tags: | + ${{ env.IMAGE }}:${{ steps.version.outputs.full }} + ${{ env.IMAGE }}:${{ steps.version.outputs.minor }} + ${{ env.IMAGE }}:${{ steps.version.outputs.major }} + ${{ env.IMAGE }}:latest + cache-from: type=gha + cache-to: type=gha,mode=max From b9301e23ce18512718c6186165681d8ebac9a647 Mon Sep 17 00:00:00 2001 From: sametd Date: Wed, 15 Apr 2026 19:46:59 +0200 Subject: [PATCH 3/7] ci: rewrite CI to use container jobs for all Linux runners Split matrix jobs into Linux (container) + macOS (bare metal). 5 Linux jobs now run inside eccr.ecmwf.int/tensogram/ci:1.0.0, eliminating LLVM, cmake, apt-get, rust-toolchain, setup-uv, setup-node, and wasm-pack installation steps. - Python installed at runtime via uv python install (~10s) - Node.js downloaded at runtime for WASM job (~3s) - Swatinem/rust-cache removed (incompatible with self-hosted runners) - Cleanup steps kept on all jobs (root-owned files on self-hosted) - permissions: contents: read for least privilege - python-free-threaded/python-packages now gate on main-linux only --- .github/workflows/ci.yml | 241 +++++++++++++++++++-------------------- 1 file changed, 119 insertions(+), 122 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6ae6703a..22e2eb33 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,69 +2,94 @@ name: CI on: push: - branches: - - main + branches: [main] pull_request: concurrency: group: ci-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +permissions: + contents: read + env: CARGO_TERM_COLOR: always CARGO_INCREMENTAL: 0 TMPDIR: ${{ github.workspace }}/.tmp - RUSTUP_HOME: ${{ github.workspace }}/.rustup - CARGO_HOME: ${{ github.workspace }}/.cargo + NODE_VERSION: "22.22.2" jobs: - # ── Main: Rust lint/test + Python bindings + GRIB + NetCDF ────────── - main: - name: Main (${{ matrix.runner }}) - runs-on: ${{ matrix.runner }} - strategy: - fail-fast: false - matrix: - runner: [[self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04], [self-hosted, macOS, platform-builder-MacOSX-13.4.1-arm64]] + # ── Linux: lint + test + GRIB + NetCDF + Python ──────────────── + main-linux: + name: Main (Linux) + runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04] + container: + image: eccr.ecmwf.int/tensogram/ci:1.0.0 steps: - uses: actions/checkout@v4 - run: mkdir -p "$TMPDIR" - - uses: dtolnay/rust-toolchain@stable - with: - components: clippy, rustfmt - - uses: Swatinem/rust-cache@v2 - - uses: astral-sh/setup-uv@v5 - with: - enable-cache: true + - name: Format check + run: cargo fmt --check - - name: Install LLVM (Linux) - if: runner.os == 'Linux' - uses: KyleMayes/install-llvm-action@v2 - with: - version: "14" - directory: ${{ github.workspace }}/.llvm + - name: Clippy + run: | + cargo clippy --workspace --all-targets -- -D warnings + cargo clippy -p tensogram-core -p tensogram-ffi -p tensogram-cli --all-targets --all-features -- -D warnings + + - name: Test + run: cargo test --workspace + + - name: Test (remote feature) + run: cargo test -p tensogram-core --features remote + + - name: Test (remote + async features) + run: cargo test -p tensogram-core --features "remote,async" + + - name: Test (GRIB) + run: | + cargo test --manifest-path crates/tensogram-grib/Cargo.toml + cargo test -p tensogram-cli --features grib - - name: Install system dependencies (Linux) - if: runner.os == 'Linux' - id: linux-deps + - name: Test (NetCDF) run: | - echo "LIBCLANG_PATH=$LLVM_PATH/lib" >> "$GITHUB_ENV" + cargo clippy --manifest-path crates/tensogram-netcdf/Cargo.toml --all-targets -- -D warnings + cargo test --manifest-path crates/tensogram-netcdf/Cargo.toml + cargo test -p tensogram-cli --features netcdf - if [ "$(id -u)" = "0" ]; then APT="apt-get"; else APT="sudo -n apt-get"; fi - install_apt() { $APT update -o DPkg::Lock::Timeout=120 && $APT install -y -o DPkg::Lock::Timeout=120 "$@"; } + - name: Build and test Python bindings + run: | + uv python install 3.13 + uv venv .venv --python 3.13 + source .venv/bin/activate + uv pip install maturin pytest pytest-asyncio numpy ruff + cd crates/tensogram-python && maturin develop && cd ../.. + ruff check --config crates/tensogram-python/pyproject.toml tests/python/ + python -m pytest tests/python/ -v + cargo check --manifest-path crates/tensogram-python/Cargo.toml --no-default-features - has_grib_deps=true - for pkg in libnetcdf-dev libhdf5-dev libeccodes-dev; do - dpkg -s "$pkg" >/dev/null 2>&1 || install_apt "$pkg" 2>/dev/null || has_grib_deps=false - done - echo "has_grib_deps=$has_grib_deps" >> "$GITHUB_OUTPUT" + - name: Cleanup + if: always() + run: rm -rf "$TMPDIR" .venv target build + + # ── macOS: lint + test + GRIB + NetCDF + Python ──────────────── + main-macos: + name: Main (macOS) + runs-on: [self-hosted, macOS, platform-builder-MacOSX-13.4.1-arm64] + env: + RUSTUP_HOME: ${{ github.workspace }}/.rustup + CARGO_HOME: ${{ github.workspace }}/.cargo + steps: + - uses: actions/checkout@v4 + - run: mkdir -p "$TMPDIR" + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy, rustfmt - which cmake >/dev/null 2>&1 && cmake --version | grep -q "3\.\(2[6-9]\|[3-9][0-9]\)" \ - || (python3 -m pip install --user cmake 2>/dev/null && echo "$HOME/.local/bin" >> "$GITHUB_PATH") \ - || (pip install cmake && echo "$HOME/.local/bin" >> "$GITHUB_PATH") + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: true - - name: Install system dependencies (macOS) - if: runner.os == 'macOS' + - name: Install system dependencies run: | for pkg in netcdf hdf5 eccodes; do brew list --formula "$pkg" >/dev/null 2>&1 || brew install "$pkg" @@ -78,9 +103,7 @@ jobs: - name: Clippy run: | cargo clippy --workspace --all-targets -- -D warnings - if [ "${{ steps.linux-deps.outputs.has_grib_deps }}" != "false" ]; then - cargo clippy -p tensogram-core -p tensogram-ffi -p tensogram-cli --all-targets --all-features -- -D warnings - fi + cargo clippy -p tensogram-core -p tensogram-ffi -p tensogram-cli --all-targets --all-features -- -D warnings - name: Test run: cargo test --workspace @@ -92,13 +115,11 @@ jobs: run: cargo test -p tensogram-core --features "remote,async" - name: Test (GRIB) - if: runner.os != 'Linux' || steps.linux-deps.outputs.has_grib_deps == 'true' run: | cargo test --manifest-path crates/tensogram-grib/Cargo.toml cargo test -p tensogram-cli --features grib - name: Test (NetCDF) - if: runner.os != 'Linux' || steps.linux-deps.outputs.has_grib_deps == 'true' run: | cargo clippy --manifest-path crates/tensogram-netcdf/Cargo.toml --all-targets -- -D warnings cargo test --manifest-path crates/tensogram-netcdf/Cargo.toml @@ -119,32 +140,17 @@ jobs: if: always() run: rm -rf "$TMPDIR" .venv build .rustup .cargo .llvm target - # ── Python: free-threaded 3.13t ─────────────────────────────────────── + # ── Python: free-threaded 3.13t ──────────────────────────────── python-free-threaded: name: Python 3.13t (free-threaded) - needs: main + needs: main-linux runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04] + container: + image: eccr.ecmwf.int/tensogram/ci:1.0.0 steps: - uses: actions/checkout@v4 - run: mkdir -p "$TMPDIR" - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - uses: astral-sh/setup-uv@v5 - with: - enable-cache: true - - - name: Install LLVM - uses: KyleMayes/install-llvm-action@v2 - with: - version: "14" - directory: ${{ github.workspace }}/.llvm - - name: Install system dependencies - run: | - echo "LIBCLANG_PATH=$LLVM_PATH/lib" >> "$GITHUB_ENV" - which cmake >/dev/null 2>&1 && cmake --version | grep -q "3\.\(2[6-9]\|[3-9][0-9]\)" \ - || (python3 -m pip install --user cmake 2>/dev/null && echo "$HOME/.local/bin" >> "$GITHUB_PATH") \ - || (pip install cmake && echo "$HOME/.local/bin" >> "$GITHUB_PATH") - name: Build and test env: @@ -160,34 +166,19 @@ jobs: - name: Cleanup if: always() - run: rm -rf "$TMPDIR" .venv .rustup .cargo .llvm target + run: rm -rf "$TMPDIR" .venv target - # ── Python packages: xarray + zarr ─────────────────────────────────── + # ── Python packages: xarray + zarr ───────────────────────────── python-packages: name: Python packages - needs: main + needs: main-linux runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04] + container: + image: eccr.ecmwf.int/tensogram/ci:1.0.0 steps: - uses: actions/checkout@v4 - run: mkdir -p "$TMPDIR" - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - uses: astral-sh/setup-uv@v5 - with: - enable-cache: true - - - name: Install LLVM - uses: KyleMayes/install-llvm-action@v2 - with: - version: "14" - directory: ${{ github.workspace }}/.llvm - - name: Install system dependencies - run: | - echo "LIBCLANG_PATH=$LLVM_PATH/lib" >> "$GITHUB_ENV" - which cmake >/dev/null 2>&1 && cmake --version | grep -q "3\.\(2[6-9]\|[3-9][0-9]\)" \ - || (python3 -m pip install --user cmake 2>/dev/null && echo "$HOME/.local/bin" >> "$GITHUB_PATH") \ - || (pip install cmake && echo "$HOME/.local/bin" >> "$GITHUB_PATH") - name: Build and test run: | @@ -203,36 +194,42 @@ jobs: - name: Cleanup if: always() - run: rm -rf "$TMPDIR" .venv .rustup .cargo .llvm target - - # ── C++: build + test ──────────────────────────────────────────────── - cpp: - name: C++ (${{ matrix.runner }}) - runs-on: ${{ matrix.runner }} - strategy: - fail-fast: false - matrix: - runner: [[self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04], [self-hosted, macOS, platform-builder-MacOSX-13.4.1-arm64]] + run: rm -rf "$TMPDIR" .venv target + + # ── C++: Linux ───────────────────────────────────────────────── + cpp-linux: + name: C++ (Linux) + runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04] + container: + image: eccr.ecmwf.int/tensogram/ci:1.0.0 steps: - uses: actions/checkout@v4 - run: mkdir -p "$TMPDIR" - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - name: Install LLVM (Linux) - if: runner.os == 'Linux' - uses: KyleMayes/install-llvm-action@v2 - with: - version: "14" - directory: ${{ github.workspace }}/.llvm - - name: Install system dependencies (Linux) - if: runner.os == 'Linux' + - name: Build and test run: | - echo "LIBCLANG_PATH=$LLVM_PATH/lib" >> "$GITHUB_ENV" - which cmake >/dev/null 2>&1 && cmake --version | grep -q "3\.\(2[6-9]\|[3-9][0-9]\)" \ - || (python3 -m pip install --user cmake 2>/dev/null && echo "$HOME/.local/bin" >> "$GITHUB_PATH") \ - || (pip install cmake && echo "$HOME/.local/bin" >> "$GITHUB_PATH") + cargo build --release -p tensogram-ffi + cmake -B build -DCMAKE_BUILD_TYPE=Release + cmake --build build -j + ctest --test-dir build --output-on-failure + + - name: Cleanup + if: always() + run: rm -rf "$TMPDIR" build target + + # ── C++: macOS ───────────────────────────────────────────────── + cpp-macos: + name: C++ (macOS) + runs-on: [self-hosted, macOS, platform-builder-MacOSX-13.4.1-arm64] + env: + RUSTUP_HOME: ${{ github.workspace }}/.rustup + CARGO_HOME: ${{ github.workspace }}/.cargo + steps: + - uses: actions/checkout@v4 + - run: mkdir -p "$TMPDIR" + - uses: dtolnay/rust-toolchain@stable + - name: Build and test run: | @@ -243,27 +240,27 @@ jobs: - name: Cleanup if: always() - run: rm -rf "$TMPDIR" build .rustup .cargo .llvm target + run: rm -rf "$TMPDIR" build .rustup .cargo target - # ── WASM: build + test ─────────────────────────────────────────────── + # ── WASM: build + test ──────────────────────────────────────── wasm: name: WASM runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04] + container: + image: eccr.ecmwf.int/tensogram/ci:1.0.0 steps: - uses: actions/checkout@v4 - run: mkdir -p "$TMPDIR" - - uses: dtolnay/rust-toolchain@stable - with: - targets: wasm32-unknown-unknown - - uses: Swatinem/rust-cache@v2 - - uses: actions/setup-node@v4 - with: - node-version: 20 - - name: Build and test + + + - name: Install Node.js run: | - which wasm-pack >/dev/null 2>&1 || cargo install wasm-pack --locked - wasm-pack test --node crates/tensogram-wasm + curl -fsSL "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.gz" \ + | tar xz --strip-components=1 -C /usr/local "node-v${NODE_VERSION}-linux-x64/bin/node" + + - name: Build and test + run: wasm-pack test --node crates/tensogram-wasm - name: Cleanup if: always() - run: rm -rf "$TMPDIR" .rustup .cargo target + run: rm -rf "$TMPDIR" target From 4ad4b43cbc472c1b0d68e7f97abf591c88fdb497 Mon Sep 17 00:00:00 2001 From: sametd Date: Wed, 15 Apr 2026 20:01:46 +0200 Subject: [PATCH 4/7] ci: add workflow_dispatch trigger to CI workflow --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22e2eb33..afdd1ab6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,7 @@ on: push: branches: [main] pull_request: + workflow_dispatch: concurrency: group: ci-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} From 4903d0febe7b60c5cc79d078b23ed8896fe40d73 Mon Sep 17 00:00:00 2001 From: sametd Date: Wed, 15 Apr 2026 20:29:07 +0200 Subject: [PATCH 5/7] ci: add registry credentials for container image pulls --- .github/workflows/ci.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afdd1ab6..60bdc61e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,6 +26,9 @@ jobs: runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04] container: image: eccr.ecmwf.int/tensogram/ci:1.0.0 + credentials: + username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} + password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} steps: - uses: actions/checkout@v4 - run: mkdir -p "$TMPDIR" @@ -148,6 +151,9 @@ jobs: runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04] container: image: eccr.ecmwf.int/tensogram/ci:1.0.0 + credentials: + username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} + password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} steps: - uses: actions/checkout@v4 - run: mkdir -p "$TMPDIR" @@ -176,6 +182,9 @@ jobs: runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04] container: image: eccr.ecmwf.int/tensogram/ci:1.0.0 + credentials: + username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} + password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} steps: - uses: actions/checkout@v4 - run: mkdir -p "$TMPDIR" @@ -203,6 +212,9 @@ jobs: runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04] container: image: eccr.ecmwf.int/tensogram/ci:1.0.0 + credentials: + username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} + password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} steps: - uses: actions/checkout@v4 - run: mkdir -p "$TMPDIR" @@ -249,6 +261,9 @@ jobs: runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04] container: image: eccr.ecmwf.int/tensogram/ci:1.0.0 + credentials: + username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} + password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} steps: - uses: actions/checkout@v4 - run: mkdir -p "$TMPDIR" From 80ecc243afe4e78248efc5ee7986056da183dd41 Mon Sep 17 00:00:00 2001 From: sametd Date: Wed, 15 Apr 2026 20:52:48 +0200 Subject: [PATCH 6/7] ci: split lint/test/python into parallel jobs and fix shell compatibility Split main-linux into 3 parallel container jobs: - lint: fmt + focused clippy (no wasted test-mode compilation) - test: all cargo test runs in optimal artifact-reuse order - python: bindings build + pytest (separate build world) Also fixes: - 'source' -> '.' for POSIX sh compatibility in containers - Over-broad '--all-features' clippy replaced with targeted feature checks per crate (avoids needless feature unification) --- .github/workflows/ci.yml | 84 ++++++++++++++++++++++++++-------------- 1 file changed, 56 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 60bdc61e..fbb89b59 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,9 +20,9 @@ env: NODE_VERSION: "22.22.2" jobs: - # ── Linux: lint + test + GRIB + NetCDF + Python ──────────────── - main-linux: - name: Main (Linux) + # ── Linux: lint (parallel with test) ─────────────────────────── + lint: + name: Lint runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04] container: image: eccr.ecmwf.int/tensogram/ci:1.0.0 @@ -32,39 +32,65 @@ jobs: steps: - uses: actions/checkout@v4 - run: mkdir -p "$TMPDIR" - - name: Format check - run: cargo fmt --check - - name: Clippy + - name: Format + Clippy run: | + cargo fmt --check cargo clippy --workspace --all-targets -- -D warnings - cargo clippy -p tensogram-core -p tensogram-ffi -p tensogram-cli --all-targets --all-features -- -D warnings - - - name: Test - run: cargo test --workspace + cargo clippy -p tensogram-core --all-targets --features "remote,async" -- -D warnings + cargo clippy -p tensogram-cli --all-targets --features "grib,netcdf" -- -D warnings + cargo clippy --manifest-path crates/tensogram-netcdf/Cargo.toml --all-targets -- -D warnings - - name: Test (remote feature) - run: cargo test -p tensogram-core --features remote + - name: Cleanup + if: always() + run: rm -rf "$TMPDIR" target - - name: Test (remote + async features) - run: cargo test -p tensogram-core --features "remote,async" + # ── Linux: test (parallel with lint) ─────────────────────────── + test: + name: Test + runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04] + container: + image: eccr.ecmwf.int/tensogram/ci:1.0.0 + credentials: + username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} + password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} + steps: + - uses: actions/checkout@v4 + - run: mkdir -p "$TMPDIR" - - name: Test (GRIB) + - name: Test run: | + cargo test --workspace + cargo test -p tensogram-core --features remote + cargo test -p tensogram-core --features "remote,async" cargo test --manifest-path crates/tensogram-grib/Cargo.toml cargo test -p tensogram-cli --features grib - - - name: Test (NetCDF) - run: | - cargo clippy --manifest-path crates/tensogram-netcdf/Cargo.toml --all-targets -- -D warnings cargo test --manifest-path crates/tensogram-netcdf/Cargo.toml cargo test -p tensogram-cli --features netcdf - - name: Build and test Python bindings + - name: Cleanup + if: always() + run: rm -rf "$TMPDIR" target + + # ── Linux: Python bindings ───────────────────────────────────── + python: + name: Python + needs: lint + runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04] + container: + image: eccr.ecmwf.int/tensogram/ci:1.0.0 + credentials: + username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} + password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} + steps: + - uses: actions/checkout@v4 + - run: mkdir -p "$TMPDIR" + + - name: Build and test run: | uv python install 3.13 uv venv .venv --python 3.13 - source .venv/bin/activate + . .venv/bin/activate uv pip install maturin pytest pytest-asyncio numpy ruff cd crates/tensogram-python && maturin develop && cd ../.. ruff check --config crates/tensogram-python/pyproject.toml tests/python/ @@ -73,7 +99,7 @@ jobs: - name: Cleanup if: always() - run: rm -rf "$TMPDIR" .venv target build + run: rm -rf "$TMPDIR" .venv target # ── macOS: lint + test + GRIB + NetCDF + Python ──────────────── main-macos: @@ -107,7 +133,9 @@ jobs: - name: Clippy run: | cargo clippy --workspace --all-targets -- -D warnings - cargo clippy -p tensogram-core -p tensogram-ffi -p tensogram-cli --all-targets --all-features -- -D warnings + cargo clippy -p tensogram-core --all-targets --features "remote,async" -- -D warnings + cargo clippy -p tensogram-cli --all-targets --features "grib,netcdf" -- -D warnings + cargo clippy --manifest-path crates/tensogram-netcdf/Cargo.toml --all-targets -- -D warnings - name: Test run: cargo test --workspace @@ -133,7 +161,7 @@ jobs: run: | uv python install 3.13 uv venv .venv --python 3.13 - source .venv/bin/activate + . .venv/bin/activate uv pip install maturin pytest pytest-asyncio numpy ruff cd crates/tensogram-python && maturin develop && cd ../.. ruff check --config crates/tensogram-python/pyproject.toml tests/python/ @@ -147,7 +175,7 @@ jobs: # ── Python: free-threaded 3.13t ──────────────────────────────── python-free-threaded: name: Python 3.13t (free-threaded) - needs: main-linux + needs: [test, python] runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04] container: image: eccr.ecmwf.int/tensogram/ci:1.0.0 @@ -165,7 +193,7 @@ jobs: run: | uv python install cpython-3.13+freethreaded uv venv .venv --python python3.13t - source .venv/bin/activate + . .venv/bin/activate uv pip install maturin pytest pytest-asyncio "numpy>=2.1" python -c "import sys; assert not sys._is_gil_enabled(), 'GIL should be disabled'" cd crates/tensogram-python && maturin develop --release && cd ../.. @@ -178,7 +206,7 @@ jobs: # ── Python packages: xarray + zarr ───────────────────────────── python-packages: name: Python packages - needs: main-linux + needs: [test, python] runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04] container: image: eccr.ecmwf.int/tensogram/ci:1.0.0 @@ -194,7 +222,7 @@ jobs: run: | uv python install 3.12 uv venv .venv --python 3.12 - source .venv/bin/activate + . .venv/bin/activate uv pip install maturin pytest numpy cd crates/tensogram-python && maturin develop && cd ../.. uv pip install -e "tensogram-xarray/[dask]" From 69f5bec8a33d3ae7f5dc1fc209e0cf1c28672bbe Mon Sep 17 00:00:00 2001 From: sametd Date: Wed, 15 Apr 2026 21:34:58 +0200 Subject: [PATCH 7/7] ci: move GRIB/NetCDF testing to macOS only, shrink image to 1.44 GB Remove libnetcdf-dev, libhdf5-dev, libeccodes-dev from the Docker image since GRIB/NetCDF are already tested on macOS. Merges python-free-threaded and python-packages into the test job to reuse compiled artifacts. Bumps image to 1.1.0. --- .github/ci/Dockerfile | 2 - .github/ci/VERSION | 2 +- .github/workflows/ci.yml | 122 ++++++++++----------------------------- 3 files changed, 32 insertions(+), 94 deletions(-) diff --git a/.github/ci/Dockerfile b/.github/ci/Dockerfile index d928b47b..2211ab79 100644 --- a/.github/ci/Dockerfile +++ b/.github/ci/Dockerfile @@ -55,9 +55,7 @@ RUN set -eux \ build-essential pkg-config git curl ca-certificates \ cmake \ libclang1-18 libclang-common-18-dev \ - libnetcdf-dev libhdf5-dev libeccodes-dev \ && rm -f /usr/lib/x86_64-linux-gnu/libicu*.a \ - /usr/lib/x86_64-linux-gnu/libhdf5*.a \ /usr/lib/x86_64-linux-gnu/libssl.a \ /usr/lib/x86_64-linux-gnu/libcrypto.a \ /usr/lib/x86_64-linux-gnu/libxml2.a \ diff --git a/.github/ci/VERSION b/.github/ci/VERSION index 3eefcb9d..9084fa2f 100644 --- a/.github/ci/VERSION +++ b/.github/ci/VERSION @@ -1 +1 @@ -1.0.0 +1.1.0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fbb89b59..604f9602 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: name: Lint runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04] container: - image: eccr.ecmwf.int/tensogram/ci:1.0.0 + image: eccr.ecmwf.int/tensogram/ci:1.1.0 credentials: username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} @@ -38,8 +38,6 @@ jobs: cargo fmt --check cargo clippy --workspace --all-targets -- -D warnings cargo clippy -p tensogram-core --all-targets --features "remote,async" -- -D warnings - cargo clippy -p tensogram-cli --all-targets --features "grib,netcdf" -- -D warnings - cargo clippy --manifest-path crates/tensogram-netcdf/Cargo.toml --all-targets -- -D warnings - name: Cleanup if: always() @@ -50,7 +48,7 @@ jobs: name: Test runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04] container: - image: eccr.ecmwf.int/tensogram/ci:1.0.0 + image: eccr.ecmwf.int/tensogram/ci:1.1.0 credentials: username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} @@ -63,30 +61,8 @@ jobs: cargo test --workspace cargo test -p tensogram-core --features remote cargo test -p tensogram-core --features "remote,async" - cargo test --manifest-path crates/tensogram-grib/Cargo.toml - cargo test -p tensogram-cli --features grib - cargo test --manifest-path crates/tensogram-netcdf/Cargo.toml - cargo test -p tensogram-cli --features netcdf - - - name: Cleanup - if: always() - run: rm -rf "$TMPDIR" target - - # ── Linux: Python bindings ───────────────────────────────────── - python: - name: Python - needs: lint - runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04] - container: - image: eccr.ecmwf.int/tensogram/ci:1.0.0 - credentials: - username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} - password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} - steps: - - uses: actions/checkout@v4 - - run: mkdir -p "$TMPDIR" - - name: Build and test + - name: Python bindings (3.13) run: | uv python install 3.13 uv venv .venv --python 3.13 @@ -97,6 +73,32 @@ jobs: python -m pytest tests/python/ -v cargo check --manifest-path crates/tensogram-python/Cargo.toml --no-default-features + - name: Python packages (3.12, xarray + zarr) + run: | + rm -rf .venv + uv python install 3.12 + uv venv .venv --python 3.12 + . .venv/bin/activate + uv pip install maturin pytest numpy + cd crates/tensogram-python && maturin develop && cd ../.. + uv pip install -e "tensogram-xarray/[dask]" + python -m pytest tensogram-xarray/tests/ -v + uv pip install -e tensogram-zarr/ + python -m pytest tensogram-zarr/tests/ -v + + - name: Python free-threaded (3.13t) + env: + PYTHON_GIL: "0" + run: | + rm -rf .venv + uv python install cpython-3.13+freethreaded + uv venv .venv --python python3.13t + . .venv/bin/activate + uv pip install maturin pytest pytest-asyncio "numpy>=2.1" + python -c "import sys; assert not sys._is_gil_enabled(), 'GIL should be disabled'" + cd crates/tensogram-python && maturin develop --release && cd ../.. + python -m pytest tests/python/ -v + - name: Cleanup if: always() run: rm -rf "$TMPDIR" .venv target @@ -172,74 +174,12 @@ jobs: if: always() run: rm -rf "$TMPDIR" .venv build .rustup .cargo .llvm target - # ── Python: free-threaded 3.13t ──────────────────────────────── - python-free-threaded: - name: Python 3.13t (free-threaded) - needs: [test, python] - runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04] - container: - image: eccr.ecmwf.int/tensogram/ci:1.0.0 - credentials: - username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} - password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} - steps: - - uses: actions/checkout@v4 - - run: mkdir -p "$TMPDIR" - - - - name: Build and test - env: - PYTHON_GIL: "0" - run: | - uv python install cpython-3.13+freethreaded - uv venv .venv --python python3.13t - . .venv/bin/activate - uv pip install maturin pytest pytest-asyncio "numpy>=2.1" - python -c "import sys; assert not sys._is_gil_enabled(), 'GIL should be disabled'" - cd crates/tensogram-python && maturin develop --release && cd ../.. - python -m pytest tests/python/ -v - - - name: Cleanup - if: always() - run: rm -rf "$TMPDIR" .venv target - - # ── Python packages: xarray + zarr ───────────────────────────── - python-packages: - name: Python packages - needs: [test, python] - runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04] - container: - image: eccr.ecmwf.int/tensogram/ci:1.0.0 - credentials: - username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} - password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} - steps: - - uses: actions/checkout@v4 - - run: mkdir -p "$TMPDIR" - - - - name: Build and test - run: | - uv python install 3.12 - uv venv .venv --python 3.12 - . .venv/bin/activate - uv pip install maturin pytest numpy - cd crates/tensogram-python && maturin develop && cd ../.. - uv pip install -e "tensogram-xarray/[dask]" - python -m pytest tensogram-xarray/tests/ -v - uv pip install -e tensogram-zarr/ - python -m pytest tensogram-zarr/tests/ -v - - - name: Cleanup - if: always() - run: rm -rf "$TMPDIR" .venv target - # ── C++: Linux ───────────────────────────────────────────────── cpp-linux: name: C++ (Linux) runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04] container: - image: eccr.ecmwf.int/tensogram/ci:1.0.0 + image: eccr.ecmwf.int/tensogram/ci:1.1.0 credentials: username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} @@ -288,7 +228,7 @@ jobs: name: WASM runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04] container: - image: eccr.ecmwf.int/tensogram/ci:1.0.0 + image: eccr.ecmwf.int/tensogram/ci:1.1.0 credentials: username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }}