ci: sccache S3 caching + parallel test jobs

.github/ci/Dockerfile

@@ -10,6 +10,7 @@
 
 # ── Pinned versions (edit here to bump) ─────────────────────────
 ARG UV_VERSION=0.11.6
+ARG SCCACHE_VERSION=0.10.0
 
 FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv
 
@@ -74,11 +75,19 @@ COPY --from=builder /usr/local/cargo  /usr/local/cargo
 # ── uv (Python + Node installed at CI runtime) ─────────────────
 COPY --from=uv /uv /uvx /usr/local/bin/
 
+# ── sccache (S3-backed compilation cache) ──────────────────────
+ARG SCCACHE_VERSION
+RUN set -eux \
+ && curl -fsSL "https://github.com/mozilla/sccache/releases/download/v${SCCACHE_VERSION}/sccache-v${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz" \
+    | tar xz --strip-components=1 -C /usr/local/bin \
+        "sccache-v${SCCACHE_VERSION}-x86_64-unknown-linux-musl/sccache"
+
 # ── Smoke test ──────────────────────────────────────────────────
 RUN set -eux \
  && rustc --version && cargo --version \
  && cmake --version \
  && wasm-pack --version \
- && uv --version
+ && uv --version \
+ && sccache --version
 
 WORKDIR /workspace

.github/ci/VERSION

@@ -1 +1 @@
-1.1.0
+1.2.0

.github/workflows/ci.yml

@@ -15,17 +15,30 @@ permissions:
 
 env:
   CARGO_TERM_COLOR: always
-  CARGO_INCREMENTAL: 0
+  CARGO_INCREMENTAL: "0"
   TMPDIR: ${{ github.workspace }}/.tmp
   NODE_VERSION: "22.22.2"
+  # sccache with S3-compatible backend (ECMWF object store).
+  # AWS_* env var names are required by the S3 SDK that sccache uses
+  # internally — no actual AWS service is involved.
+  RUSTC_WRAPPER: sccache
+  SCCACHE_BUCKET: tensogram-ci-cache
+  SCCACHE_ENDPOINT: "https://object-store.os-api.cci1.ecmwf.int"
+  SCCACHE_REGION: us-east-1
+  SCCACHE_S3_USE_SSL: "true"
+  SCCACHE_S3_KEY_PREFIX: tensogram/
+  SCCACHE_BASEDIRS: ${{ github.workspace }}
+  SCCACHE_IGNORE_SERVER_IO_ERROR: "1"
+  AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
 
 jobs:
-  # ── Linux: lint (parallel with test) ───────────────────────────
+  # ── Linux: lint ────────────────────────────────────────────────
   lint:
     name: Lint
     runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04]
     container:
-      image: eccr.ecmwf.int/tensogram/ci:1.1.0
+      image: eccr.ecmwf.int/tensogram/ci:1.2.0
       credentials:
         username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }}
         password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }}
@@ -39,16 +52,20 @@ jobs:
           cargo clippy --workspace --all-targets -- -D warnings
           cargo clippy -p tensogram-core --all-targets --features "remote,async" -- -D warnings
 
+      - name: sccache stats
+        if: always()
+        run: sccache --show-adv-stats
+
       - name: Cleanup
         if: always()
         run: rm -rf "$TMPDIR" target
 
-  # ── Linux: test (parallel with lint) ───────────────────────────
-  test:
-    name: Test
+  # ── Linux: Rust tests ──────────────────────────────────────────
+  test-rust:
+    name: Test (Rust)
     runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04]
     container:
-      image: eccr.ecmwf.int/tensogram/ci:1.1.0
+      image: eccr.ecmwf.int/tensogram/ci:1.2.0
       credentials:
         username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }}
         password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }}
@@ -62,6 +79,27 @@ jobs:
           cargo test -p tensogram-core --features remote
           cargo test -p tensogram-core --features "remote,async"
 
+      - name: sccache stats
+        if: always()
+        run: sccache --show-adv-stats
+
+      - name: Cleanup
+        if: always()
+        run: rm -rf "$TMPDIR" target
+
+  # ── Linux: Python 3.13 bindings ────────────────────────────────
+  test-python:
+    name: Test (Python 3.13)
+    runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04]
+    container:
+      image: eccr.ecmwf.int/tensogram/ci:1.2.0
+      credentials:
+        username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }}
+        password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }}
+    steps:
+      - uses: actions/checkout@v4
+      - run: mkdir -p "$TMPDIR"
+
       - name: Python bindings (3.13)
         run: |
           uv python install 3.13
@@ -73,9 +111,29 @@ jobs:
           python -m pytest python/tests/ -v
           cargo check --manifest-path python/bindings/Cargo.toml --no-default-features
 
+      - name: sccache stats
+        if: always()
+        run: sccache --show-adv-stats
+
+      - name: Cleanup
+        if: always()
+        run: rm -rf "$TMPDIR" .venv target
+
+  # ── Linux: xarray + zarr (Python 3.12) ─────────────────────────
+  test-python-packages:
+    name: Test (xarray + zarr)
+    runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04]
+    container:
+      image: eccr.ecmwf.int/tensogram/ci:1.2.0
+      credentials:
+        username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }}
+        password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }}
+    steps:
+      - uses: actions/checkout@v4
+      - run: mkdir -p "$TMPDIR"
+
       - name: Python packages (3.12, xarray + zarr)
         run: |
-          rm -rf .venv
           uv python install 3.12
           uv venv .venv --python 3.12
           . .venv/bin/activate
@@ -86,11 +144,31 @@ jobs:
           uv pip install -e python/tensogram-zarr/
           python -m pytest python/tensogram-zarr/tests/ -v
 
+      - name: sccache stats
+        if: always()
+        run: sccache --show-adv-stats
+
+      - name: Cleanup
+        if: always()
+        run: rm -rf "$TMPDIR" .venv target
+
+  # ── Linux: free-threaded Python 3.13t ──────────────────────────
+  test-python-ft:
+    name: Test (Python 3.13t free-threaded)
+    runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04]
+    container:
+      image: eccr.ecmwf.int/tensogram/ci:1.2.0
+      credentials:
+        username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }}
+        password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }}
+    steps:
+      - uses: actions/checkout@v4
+      - run: mkdir -p "$TMPDIR"
+
       - name: Python free-threaded (3.13t)
         env:
           PYTHON_GIL: "0"
         run: |
-          rm -rf .venv
           uv python install cpython-3.13+freethreaded
           uv venv .venv --python python3.13t
           . .venv/bin/activate
@@ -99,6 +177,10 @@ jobs:
           cd python/bindings && maturin develop --release && cd ../..
           python -m pytest python/tests/ -v
 
+      - name: sccache stats
+        if: always()
+        run: sccache --show-adv-stats
+
       - name: Cleanup
         if: always()
         run: rm -rf "$TMPDIR" .venv target
@@ -110,6 +192,7 @@ jobs:
     env:
       RUSTUP_HOME: ${{ github.workspace }}/.rustup
       CARGO_HOME: ${{ github.workspace }}/.cargo
+      SCCACHE_DIR: ${{ github.workspace }}/.sccache
     steps:
       - uses: actions/checkout@v4
       - run: mkdir -p "$TMPDIR"
@@ -121,6 +204,9 @@ jobs:
         with:
           enable-cache: true
 
+      - name: Install sccache
+        run: brew list sccache &>/dev/null || brew install sccache
+
       - name: Install system dependencies
         run: |
           for pkg in netcdf hdf5 eccodes; do
@@ -170,31 +256,42 @@ jobs:
           python -m pytest python/tests/ -v
           cargo check --manifest-path python/bindings/Cargo.toml --no-default-features
 
+      - name: sccache stats
+        if: always()
+        run: sccache --show-adv-stats || true
+
       - name: Cleanup
         if: always()
-        run: rm -rf "$TMPDIR" .venv build .rustup .cargo .llvm target
+        run: |
+          sccache --stop-server || true
+          rm -rf "$TMPDIR" .venv build .rustup .cargo .sccache .llvm target
 
   # ── C++: Linux ─────────────────────────────────────────────────
   cpp-linux:
     name: C++ (Linux)
     runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04]
     container:
-      image: eccr.ecmwf.int/tensogram/ci:1.1.0
+      image: eccr.ecmwf.int/tensogram/ci:1.2.0
       credentials:
         username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }}
         password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }}
     steps:
       - uses: actions/checkout@v4
       - run: mkdir -p "$TMPDIR"
 
-
       - name: Build and test
         run: |
           cargo build --release -p tensogram-ffi
-          cmake -S cpp -B build -DCMAKE_BUILD_TYPE=Release
+          cmake -S cpp -B build -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_C_COMPILER_LAUNCHER=sccache \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
           cmake --build build -j
           ctest --test-dir build --output-on-failure
 
+      - name: sccache stats
+        if: always()
+        run: sccache --show-adv-stats
+
       - name: Cleanup
         if: always()
         run: rm -rf "$TMPDIR" build target
@@ -206,37 +303,47 @@ jobs:
     env:
       RUSTUP_HOME: ${{ github.workspace }}/.rustup
       CARGO_HOME: ${{ github.workspace }}/.cargo
+      SCCACHE_DIR: ${{ github.workspace }}/.sccache
     steps:
       - uses: actions/checkout@v4
       - run: mkdir -p "$TMPDIR"
       - uses: dtolnay/rust-toolchain@stable
 
+      - name: Install sccache
+        run: brew list sccache &>/dev/null || brew install sccache
 
       - name: Build and test
         run: |
           cargo build --release -p tensogram-ffi
-          cmake -S cpp -B build -DCMAKE_BUILD_TYPE=Release
+          cmake -S cpp -B build -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_C_COMPILER_LAUNCHER=sccache \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
           cmake --build build -j
           ctest --test-dir build --output-on-failure
 
+      - name: sccache stats
+        if: always()
+        run: sccache --show-adv-stats || true
+
       - name: Cleanup
         if: always()
-        run: rm -rf "$TMPDIR" build .rustup .cargo target
+        run: |
+          sccache --stop-server || true
+          rm -rf "$TMPDIR" build .rustup .cargo .sccache target
 
   # ── WASM: build + test ────────────────────────────────────────
   wasm:
     name: WASM
     runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04]
     container:
-      image: eccr.ecmwf.int/tensogram/ci:1.1.0
+      image: eccr.ecmwf.int/tensogram/ci:1.2.0
       credentials:
         username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }}
         password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }}
     steps:
       - uses: actions/checkout@v4
       - run: mkdir -p "$TMPDIR"
 
-
       - name: Install Node.js
         run: |
           curl -fsSL "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.gz" \
@@ -245,6 +352,10 @@ jobs:
       - name: Build and test
         run: wasm-pack test --node rust/tensogram-wasm
 
+      - name: sccache stats
+        if: always()
+        run: sccache --show-adv-stats
+
       - name: Cleanup
         if: always()
         run: rm -rf "$TMPDIR" target
@@ -254,7 +365,7 @@ jobs:
     name: TypeScript
     runs-on: [self-hosted, Linux, platform-builder-docker-xl, platform-builder-Ubuntu-22.04]
     container:
-      image: eccr.ecmwf.int/tensogram/ci:1.1.0
+      image: eccr.ecmwf.int/tensogram/ci:1.2.0
       credentials:
         username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }}
         password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }}
@@ -299,6 +410,10 @@ jobs:
           npx tsx 06_file_api.ts
           npx tsx 07_hash_and_errors.ts
 
+      - name: sccache stats
+        if: always()
+        run: sccache --show-adv-stats
+
       - name: Cleanup
         if: always()
         run: rm -rf "$TMPDIR" target typescript/node_modules typescript/dist typescript/wasm examples/typescript/node_modules