From 08cad30290de580d1328afd1604f3c0b38cbb4e6 Mon Sep 17 00:00:00 2001 From: sametd Date: Fri, 17 Apr 2026 23:35:52 +0200 Subject: [PATCH 1/2] ci: add sccache to CI Docker image for S3-backed compilation caching --- .github/ci/Dockerfile | 11 ++++++++++- .github/ci/VERSION | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/ci/Dockerfile b/.github/ci/Dockerfile index 2211ab79..7d74773d 100644 --- a/.github/ci/Dockerfile +++ b/.github/ci/Dockerfile @@ -10,6 +10,7 @@ # ── Pinned versions (edit here to bump) ───────────────────────── ARG UV_VERSION=0.11.6 +ARG SCCACHE_VERSION=0.10.0 FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv @@ -74,11 +75,19 @@ COPY --from=builder /usr/local/cargo /usr/local/cargo # ── uv (Python + Node installed at CI runtime) ───────────────── COPY --from=uv /uv /uvx /usr/local/bin/ +# ── sccache (S3-backed compilation cache) ────────────────────── +ARG SCCACHE_VERSION +RUN set -eux \ + && curl -fsSL "https://github.com/mozilla/sccache/releases/download/v${SCCACHE_VERSION}/sccache-v${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz" \ + | tar xz --strip-components=1 -C /usr/local/bin \ + "sccache-v${SCCACHE_VERSION}-x86_64-unknown-linux-musl/sccache" + # ── Smoke test ────────────────────────────────────────────────── RUN set -eux \ && rustc --version && cargo --version \ && cmake --version \ && wasm-pack --version \ - && uv --version + && uv --version \ + && sccache --version WORKDIR /workspace diff --git a/.github/ci/VERSION b/.github/ci/VERSION index 9084fa2f..26aaba0e 100644 --- a/.github/ci/VERSION +++ b/.github/ci/VERSION @@ -1 +1 @@ -1.1.0 +1.2.0 From fa72afe0230e91867eba465f0d07fe093cdc4cd7 Mon Sep 17 00:00:00 2001 From: sametd Date: Fri, 17 Apr 2026 23:35:59 +0200 Subject: [PATCH 2/2] ci: enable sccache S3 caching and split test job into parallel jobs Configure sccache with ECMWF S3-compatible object store for shared compilation caching across all CI jobs. Split the monolithic test job into four parallel jobs (Rust, Python 3.13, xarray+zarr, free-threaded) to reduce the Linux critical path. Add sccache stats reporting and cmake compiler launchers for C++ caching. --- .github/workflows/ci.yml | 151 ++++++++++++++++++++++++++++++++++----- 1 file changed, 133 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 93e16011..a41f5cc7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,17 +15,30 @@ permissions: env: CARGO_TERM_COLOR: always - CARGO_INCREMENTAL: 0 + CARGO_INCREMENTAL: "0" TMPDIR: ${{ github.workspace }}/.tmp NODE_VERSION: "22.22.2" + # sccache with S3-compatible backend (ECMWF object store). + # AWS_* env var names are required by the S3 SDK that sccache uses + # internally — no actual AWS service is involved. + RUSTC_WRAPPER: sccache + SCCACHE_BUCKET: tensogram-ci-cache + SCCACHE_ENDPOINT: "https://object-store.os-api.cci1.ecmwf.int" + SCCACHE_REGION: us-east-1 + SCCACHE_S3_USE_SSL: "true" + SCCACHE_S3_KEY_PREFIX: tensogram/ + SCCACHE_BASEDIRS: ${{ github.workspace }} + SCCACHE_IGNORE_SERVER_IO_ERROR: "1" + AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }} jobs: - # ── Linux: lint (parallel with test) ─────────────────────────── + # ── Linux: lint ──────────────────────────────────────────────── lint: name: Lint runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04] container: - image: eccr.ecmwf.int/tensogram/ci:1.1.0 + image: eccr.ecmwf.int/tensogram/ci:1.2.0 credentials: username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} @@ -39,16 +52,20 @@ jobs: cargo clippy --workspace --all-targets -- -D warnings cargo clippy -p tensogram-core --all-targets --features "remote,async" -- -D warnings + - name: sccache stats + if: always() + run: sccache --show-adv-stats + - name: Cleanup if: always() run: rm -rf "$TMPDIR" target - # ── Linux: test (parallel with lint) ─────────────────────────── - test: - name: Test + # ── Linux: Rust tests ────────────────────────────────────────── + test-rust: + name: Test (Rust) runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04] container: - image: eccr.ecmwf.int/tensogram/ci:1.1.0 + image: eccr.ecmwf.int/tensogram/ci:1.2.0 credentials: username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} @@ -62,6 +79,27 @@ jobs: cargo test -p tensogram-core --features remote cargo test -p tensogram-core --features "remote,async" + - name: sccache stats + if: always() + run: sccache --show-adv-stats + + - name: Cleanup + if: always() + run: rm -rf "$TMPDIR" target + + # ── Linux: Python 3.13 bindings ──────────────────────────────── + test-python: + name: Test (Python 3.13) + runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04] + container: + image: eccr.ecmwf.int/tensogram/ci:1.2.0 + credentials: + username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} + password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} + steps: + - uses: actions/checkout@v4 + - run: mkdir -p "$TMPDIR" + - name: Python bindings (3.13) run: | uv python install 3.13 @@ -73,9 +111,29 @@ jobs: python -m pytest python/tests/ -v cargo check --manifest-path python/bindings/Cargo.toml --no-default-features + - name: sccache stats + if: always() + run: sccache --show-adv-stats + + - name: Cleanup + if: always() + run: rm -rf "$TMPDIR" .venv target + + # ── Linux: xarray + zarr (Python 3.12) ───────────────────────── + test-python-packages: + name: Test (xarray + zarr) + runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04] + container: + image: eccr.ecmwf.int/tensogram/ci:1.2.0 + credentials: + username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} + password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} + steps: + - uses: actions/checkout@v4 + - run: mkdir -p "$TMPDIR" + - name: Python packages (3.12, xarray + zarr) run: | - rm -rf .venv uv python install 3.12 uv venv .venv --python 3.12 . .venv/bin/activate @@ -86,11 +144,31 @@ jobs: uv pip install -e python/tensogram-zarr/ python -m pytest python/tensogram-zarr/tests/ -v + - name: sccache stats + if: always() + run: sccache --show-adv-stats + + - name: Cleanup + if: always() + run: rm -rf "$TMPDIR" .venv target + + # ── Linux: free-threaded Python 3.13t ────────────────────────── + test-python-ft: + name: Test (Python 3.13t free-threaded) + runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04] + container: + image: eccr.ecmwf.int/tensogram/ci:1.2.0 + credentials: + username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} + password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} + steps: + - uses: actions/checkout@v4 + - run: mkdir -p "$TMPDIR" + - name: Python free-threaded (3.13t) env: PYTHON_GIL: "0" run: | - rm -rf .venv uv python install cpython-3.13+freethreaded uv venv .venv --python python3.13t . .venv/bin/activate @@ -99,6 +177,10 @@ jobs: cd python/bindings && maturin develop --release && cd ../.. python -m pytest python/tests/ -v + - name: sccache stats + if: always() + run: sccache --show-adv-stats + - name: Cleanup if: always() run: rm -rf "$TMPDIR" .venv target @@ -110,6 +192,7 @@ jobs: env: RUSTUP_HOME: ${{ github.workspace }}/.rustup CARGO_HOME: ${{ github.workspace }}/.cargo + SCCACHE_DIR: ${{ github.workspace }}/.sccache steps: - uses: actions/checkout@v4 - run: mkdir -p "$TMPDIR" @@ -121,6 +204,9 @@ jobs: with: enable-cache: true + - name: Install sccache + run: brew list sccache &>/dev/null || brew install sccache + - name: Install system dependencies run: | for pkg in netcdf hdf5 eccodes; do @@ -170,16 +256,22 @@ jobs: python -m pytest python/tests/ -v cargo check --manifest-path python/bindings/Cargo.toml --no-default-features + - name: sccache stats + if: always() + run: sccache --show-adv-stats || true + - name: Cleanup if: always() - run: rm -rf "$TMPDIR" .venv build .rustup .cargo .llvm target + run: | + sccache --stop-server || true + rm -rf "$TMPDIR" .venv build .rustup .cargo .sccache .llvm target # ── C++: Linux ───────────────────────────────────────────────── cpp-linux: name: C++ (Linux) runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04] container: - image: eccr.ecmwf.int/tensogram/ci:1.1.0 + image: eccr.ecmwf.int/tensogram/ci:1.2.0 credentials: username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} @@ -187,14 +279,19 @@ jobs: - uses: actions/checkout@v4 - run: mkdir -p "$TMPDIR" - - name: Build and test run: | cargo build --release -p tensogram-ffi - cmake -S cpp -B build -DCMAKE_BUILD_TYPE=Release + cmake -S cpp -B build -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER_LAUNCHER=sccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=sccache cmake --build build -j ctest --test-dir build --output-on-failure + - name: sccache stats + if: always() + run: sccache --show-adv-stats + - name: Cleanup if: always() run: rm -rf "$TMPDIR" build target @@ -206,29 +303,40 @@ jobs: env: RUSTUP_HOME: ${{ github.workspace }}/.rustup CARGO_HOME: ${{ github.workspace }}/.cargo + SCCACHE_DIR: ${{ github.workspace }}/.sccache steps: - uses: actions/checkout@v4 - run: mkdir -p "$TMPDIR" - uses: dtolnay/rust-toolchain@stable + - name: Install sccache + run: brew list sccache &>/dev/null || brew install sccache - name: Build and test run: | cargo build --release -p tensogram-ffi - cmake -S cpp -B build -DCMAKE_BUILD_TYPE=Release + cmake -S cpp -B build -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER_LAUNCHER=sccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=sccache cmake --build build -j ctest --test-dir build --output-on-failure + - name: sccache stats + if: always() + run: sccache --show-adv-stats || true + - name: Cleanup if: always() - run: rm -rf "$TMPDIR" build .rustup .cargo target + run: | + sccache --stop-server || true + rm -rf "$TMPDIR" build .rustup .cargo .sccache target # ── WASM: build + test ──────────────────────────────────────── wasm: name: WASM runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04] container: - image: eccr.ecmwf.int/tensogram/ci:1.1.0 + image: eccr.ecmwf.int/tensogram/ci:1.2.0 credentials: username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} @@ -236,7 +344,6 @@ jobs: - uses: actions/checkout@v4 - run: mkdir -p "$TMPDIR" - - name: Install Node.js run: | curl -fsSL "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.gz" \ @@ -245,6 +352,10 @@ jobs: - name: Build and test run: wasm-pack test --node rust/tensogram-wasm + - name: sccache stats + if: always() + run: sccache --show-adv-stats + - name: Cleanup if: always() run: rm -rf "$TMPDIR" target @@ -254,7 +365,7 @@ jobs: name: TypeScript runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04] container: - image: eccr.ecmwf.int/tensogram/ci:1.1.0 + image: eccr.ecmwf.int/tensogram/ci:1.2.0 credentials: username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} @@ -299,6 +410,10 @@ jobs: npx tsx 06_file_api.ts npx tsx 07_hash_and_errors.ts + - name: sccache stats + if: always() + run: sccache --show-adv-stats + - name: Cleanup if: always() run: rm -rf "$TMPDIR" target typescript/node_modules typescript/dist typescript/wasm examples/typescript/node_modules