implemented fetch/push-hpc-tree commands including tests

.github/workflows/ci.yml

@@ -5,7 +5,10 @@
 name: CI
 
 on:
+  # push scoped to main so a branch with an open PR runs the pull_request copy
+  # only, not a redundant second push run.
   push:
+    branches: [main]
   pull_request:
 
 jobs:

.github/workflows/hpc-transfer-e2e.yml

@@ -0,0 +1,190 @@
+# SPDX-FileCopyrightText: 2026 European Centre for Medium-Range Weather Forecasts (ECMWF)
+#
+# SPDX-License-Identifier: Apache-2.0
+
+name: HPC transfer e2e
+
+# Real-world usage test for the fetch-tree / push-tree primitives, in two tiers:
+#
+#   * local-direct-roundtrip (every push/PR, ubuntu-latest, no cluster) drives the
+#     commands against the local-direct troika site (connection: local), so troika
+#     runs the tar commands and getfile/sendfile on this runner — a genuine
+#     tar -> transfer -> untar round-trip with source and destination both local.
+#     Exercised through the CLI and through the composite actions.
+#   * hpc-roundtrip (manual only, self-hosted [hpc] login-node runner) does the
+#     real thing: pushes a tree to shared scratch on the cluster over troika ssh
+#     and fetches it back.
+#
+# push is scoped to main so a branch with an open PR runs the pull_request copy
+# only, not a redundant second push run.
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  local-direct-roundtrip:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
+        with:
+          python-version: "3.12"
+
+      - name: Install ci-infrastructure (this branch)
+        run: pip install -e .
+
+      # The composite actions bootstrap via ensure-infrastructure-present@main,
+      # which would install ci-infrastructure from main — without the commands
+      # this branch adds. Point CI_INFRASTRUCTURE_PYTHON at the interpreter we
+      # just installed the branch into; ensure-infrastructure-present then takes
+      # its reuse fast path and the actions run this branch's code.
+      - name: Expose the branch interpreter to the composite actions
+        shell: bash
+        run: echo "CI_INFRASTRUCTURE_PYTHON=$(python -c 'import sys; print(sys.executable)')" >> "$GITHUB_ENV"
+
+      - name: CLI round-trip through local-direct
+        shell: bash
+        run: |
+          set -euo pipefail
+          cfg=src/ci_infrastructure/hpc/troika-config.yml
+          src="$RUNNER_TEMP/cli/src"
+          mkdir -p "$src/nested"
+          echo runner-payload > "$src/hello.txt"
+          echo deep > "$src/nested/deep.txt"
+
+          python -m ci_infrastructure.hpc push-tree --site local-direct --troika-config "$cfg" \
+            --local-dir "$src" --remote-dir "$RUNNER_TEMP/cli/remote" --tar-dir "$RUNNER_TEMP/cli/tars"
+          python -m ci_infrastructure.hpc fetch-tree --site local-direct --troika-config "$cfg" \
+            --remote-dir "$RUNNER_TEMP/cli/remote" --local-dir "$RUNNER_TEMP/cli/back" --tar-dir "$RUNNER_TEMP/cli/tars"
+          diff -r "$src" "$RUNNER_TEMP/cli/back"
+
+          # A dry run resolves the path but must transfer nothing.
+          python -m ci_infrastructure.hpc fetch-tree --site local-direct --troika-config "$cfg" \
+            --remote-dir "$RUNNER_TEMP/cli/remote" --local-dir "$RUNNER_TEMP/cli/dry" \
+            --tar-dir "$RUNNER_TEMP/cli/tars" --dryrun
+          test ! -e "$RUNNER_TEMP/cli/dry"
+          echo "CLI round-trip OK"
+
+      - name: Seed a tree for the action round-trip
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p "$RUNNER_TEMP/act/src/nested"
+          echo action-payload > "$RUNNER_TEMP/act/src/hello.txt"
+          echo deep > "$RUNNER_TEMP/act/src/nested/deep.txt"
+
+      - name: Push the tree up via the action
+        id: push
+        uses: ./actions/push-hpc-tree
+        with:
+          site: local-direct
+          local-dir: ${{ runner.temp }}/act/src
+          remote-dir: ${{ runner.temp }}/act/remote
+
+      - name: Fetch the tree back via the action
+        id: fetch
+        uses: ./actions/fetch-hpc-tree
+        with:
+          site: local-direct
+          remote-dir: ${{ runner.temp }}/act/remote
+          local-dir: ${{ runner.temp }}/act/back
+
+      - name: Assert the action outputs and the round-trip
+        shell: bash
+        env:
+          PUSH_REMOTE_DIR: ${{ steps.push.outputs.remote-dir }}
+          FETCH_LOCAL_DIR: ${{ steps.fetch.outputs.local-dir }}
+        run: |
+          set -euo pipefail
+          test "$PUSH_REMOTE_DIR" = "$RUNNER_TEMP/act/remote"
+          test "$FETCH_LOCAL_DIR" = "$RUNNER_TEMP/act/back"
+          diff -r "$RUNNER_TEMP/act/src" "$RUNNER_TEMP/act/back"
+          echo "Action round-trip OK"
+
+  hpc-roundtrip:
+    # Manual only: it reaches the real cluster over troika ssh from the login-node
+    # self-hosted runner. The ubuntu-latest job above is the per-push gate.
+    if: github.event_name == 'workflow_dispatch'
+    runs-on: [hpc]
+    steps:
+      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+
+      - uses: ecmwf/ci-infrastructure/actions/ensure-infrastructure-present@main
+
+      - name: Install this checkout's ci-infrastructure into a test venv
+        shell: bash
+        run: |
+          set -euo pipefail
+          # ensure-infrastructure-present bootstraps a known-good interpreter, but it
+          # installs ci-infrastructure from @main — which pre-merge lacks fetch-tree /
+          # push-tree. Build a venv from THIS checkout so the test runs the checked-out
+          # code, and point CI_INFRASTRUCTURE_PYTHON at it; the composite actions below
+          # then take their reuse fast path and run this code.
+          if [ -z "${CI_INFRASTRUCTURE_PYTHON:-}" ]; then
+            echo "::error::CI_INFRASTRUCTURE_PYTHON is unset — ensure-infrastructure-present did not run." >&2
+            exit 1
+          fi
+          venv="$RUNNER_TEMP/transfer-e2e-venv"
+          rm -rf "$venv"
+          "$CI_INFRASTRUCTURE_PYTHON" -m venv "$venv"
+          "$venv/bin/pip" install --disable-pip-version-check -e .
+          echo "CI_INFRASTRUCTURE_PYTHON=$venv/bin/python" >> "$GITHUB_ENV"
+
+      - name: Seed a tree to push to the cluster
+        shell: bash
+        run: |
+          set -euo pipefail
+          src="$RUNNER_TEMP/hpc-e2e/src"
+          mkdir -p "$src/nested"
+          echo "hpc-e2e $GITHUB_RUN_ID" > "$src/hello.txt"
+          echo deep > "$src/nested/deep.txt"
+
+      - name: Push the tree up to the cluster
+        id: push
+        uses: ./actions/push-hpc-tree
+        with:
+          site: hpc-batch
+          troika-user: ${{ secrets.HPC_CI_SSH_USER }}
+          local-dir: ${{ runner.temp }}/hpc-e2e/src
+          remote-dir: ${{ vars.HPC_CI_REMOTE_WORK_DIR }}/transfer-e2e-${{ github.run_id }}
+
+      - name: Fetch the tree back from the cluster
+        id: fetch
+        uses: ./actions/fetch-hpc-tree
+        with:
+          site: hpc-batch
+          troika-user: ${{ secrets.HPC_CI_SSH_USER }}
+          remote-dir: ${{ steps.push.outputs.remote-dir }}
+          local-dir: ${{ runner.temp }}/hpc-e2e/back
+
+      - name: Assert the round-trip
+        shell: bash
+        run: |
+          set -euo pipefail
+          diff -r "$RUNNER_TEMP/hpc-e2e/src" "$RUNNER_TEMP/hpc-e2e/back"
+          echo "HPC round-trip OK"
+
+      - name: Remove the cluster scratch dir
+        if: always()
+        shell: bash
+        env:
+          TROIKA_USER: ${{ secrets.HPC_CI_SSH_USER }}
+          REMOTE_DIR: ${{ steps.push.outputs.remote-dir }}
+        run: |
+          set -euo pipefail
+          case "${REMOTE_DIR:-}" in
+            /*) ;;  # only proceed for an absolute path the push step resolved
+            *) echo "nothing to clean (push did not resolve a remote dir)"; exit 0 ;;
+          esac
+          [ -n "${CI_INFRASTRUCTURE_PYTHON:-}" ] || exit 0
+          # Every python line sits at this block's indent so the YAML block scalar
+          # keeps them; after the indent is stripped they are column-0 statements
+          # (a one-line comprehension, so there is no indented loop body to break it).
+          "$CI_INFRASTRUCTURE_PYTHON" -c 'import os, subprocess
+          from ci_infrastructure.hpc.site import load_site
+          conn = load_site("hpc-batch", user=(os.environ.get("TROIKA_USER") or None))._connection
+          base = os.environ["REMOTE_DIR"]
+          [conn.execute(["rm", "-rf", p], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() for p in (base, base + ".push.tgz", base + ".fetch.tgz")]
+          print("cleaned", base)'

HPC.md

@@ -163,3 +163,60 @@ The submit-then-poll path leaves per-artifact `staging/`, `src/`, `install/` and
   on the login-node runner, sweeping per-artifact trees older than `N` days. Run
   it via `workflow_dispatch` with `dryrun: true` first to see what it would
   remove.
+
+## Moving extra directories between the runner and the cluster
+
+The build flow already brackets a job with two tree transfers over troika's
+connection. The same transfer is also available standalone, for any workflow that
+needs to move a directory in or out of the cluster outside a build — e.g. pulling
+a job's reference/artifact directory back for a later processing step, or staging
+inputs onto shared scratch before a job reads them. Both directions are a plain
+login-node copy: **no scheduler and no S3**, so they work against `direct` sites
+too.
+
+- **`fetch-tree`** — cluster → runner. Tars `--remote-dir` on the cluster, brings
+  the single tarball back and unpacks it into `--local-dir` on the runner. Writes
+  the runner-local directory as the `local-dir` output.
+- **`push-tree`** — runner → cluster. Tars `--local-dir` on the runner, ships it up
+  and unpacks it into `--remote-dir` on the cluster. Writes the resolved cluster
+  directory as the `remote-dir` output.
+
+Two rules for the remote directory:
+
+- it must live on a filesystem the login node can reach (shared scratch — the same
+  Lustre `$SCRATCH` the login-node runner and the compute nodes all see);
+- `--remote-dir` is expanded **on the cluster**, so quote a `$SCRATCH/…` spec to
+  keep the runner's shell from expanding it first (same rule as `--remote-work-dir`
+  — see *Why the work dir is expanded on the cluster, not on the runner*).
+
+As composite actions (post-step to pull a job's output back to the runner):
+
+```yaml
+- uses: ecmwf/ci-infrastructure/actions/fetch-hpc-tree@main
+  with:
+    site: hpc-batch            # same troika site the job used (or lumi)
+    troika-user: ${{ secrets.HPC_CI_SSH_USER }}
+    remote-dir: ${{ env.OUTPUT_DIR }}/ectrans-reference-artifact
+    local-dir: ./ectrans-reference-artifact
+```
+
+```yaml
+- uses: ecmwf/ci-infrastructure/actions/push-hpc-tree@main
+  with:
+    site: hpc-batch
+    troika-user: ${{ secrets.HPC_CI_SSH_USER }}
+    local-dir: ./inputs
+    remote-dir: ${{ env.OUTPUT_DIR }}/inputs
+```
+
+Or directly, e.g. on the login-node runner:
+
+```bash
+python -m ci_infrastructure.hpc fetch-tree --site hpc-batch \
+  --remote-dir "$OUTPUT_DIR/ectrans-reference-artifact" \
+  --local-dir ./ref --tar-dir "$RUNNER_TEMP/hpc-tars"
+```
+
+`.github/workflows/hpc-transfer-e2e.yml` exercises both commands and both actions
+on every push against the `local-direct` site (a real tar → transfer → untar
+round-trip with no cluster).

actions/fetch-hpc-tree/action.yml

@@ -0,0 +1,78 @@
+# SPDX-FileCopyrightText: 2026 European Centre for Medium-Range Weather Forecasts (ECMWF)
+#
+# SPDX-License-Identifier: Apache-2.0
+
+name: Fetch HPC tree
+description: >
+  Copy a directory a job produced on the cluster back to the runner, over
+  troika's own connection (no scheduler, no S3). Use it as a post-step after an
+  HPC job to pull an output directory — reference data, artifacts — off shared
+  scratch and onto the runner for later processing.
+
+  The remote directory must sit on a filesystem the login node can read (shared
+  scratch). It may name cluster variables such as SCRATCH; the spec is expanded
+  on the cluster, so pass it quoted so this runner does not expand it first. The
+  runner-local extraction directory is exposed as the local-dir output.
+
+inputs:
+  site:
+    description: 'Troika site name (see troika-config.yml), e.g. hpc-batch or lumi'
+    required: true
+  remote-dir:
+    description: 'Source directory on the cluster to fetch (may name cluster variables like $SCRATCH)'
+    required: true
+  local-dir:
+    description: 'Runner-local directory to unpack the tree into'
+    required: true
+  tar-dir:
+    description: 'Runner-local scratch dir for the transferred tarball. Empty falls back to $RUNNER_TEMP/hpc-tars.'
+    required: false
+    default: ''
+  troika-config:
+    description: 'Path to a troika config. Empty uses the one packaged with ci-infrastructure.'
+    required: false
+    default: ''
+  troika-user:
+    description: 'Remote/scheduler user for troika. Empty leaves troika to resolve it.'
+    required: false
+    default: ''
+
+outputs:
+  local-dir:
+    description: 'Runner-local directory the tree was unpacked into'
+    value: ${{ steps.fetch.outputs.local-dir }}
+
+runs:
+  using: composite
+  steps:
+    - uses: ecmwf/ci-infrastructure/actions/ensure-infrastructure-present@main
+
+    - name: Fetch the tree off the cluster
+      id: fetch
+      shell: bash
+      env:
+        SITE: ${{ inputs.site }}
+        REMOTE_DIR: ${{ inputs.remote-dir }}
+        LOCAL_DIR: ${{ inputs.local-dir }}
+        TAR_DIR_INPUT: ${{ inputs.tar-dir }}
+        TROIKA_CONFIG: ${{ inputs.troika-config }}
+        TROIKA_USER: ${{ inputs.troika-user }}
+      run: |
+        set -euo pipefail
+        if [ -z "${CI_INFRASTRUCTURE_PYTHON:-}" ]; then
+          echo "::error::CI_INFRASTRUCTURE_PYTHON is unset — ensure-infrastructure-present did not run." >&2
+          exit 1
+        fi
+        tar_dir="${TAR_DIR_INPUT:-$RUNNER_TEMP/hpc-tars}"
+        mkdir -p "$tar_dir"
+        # REMOTE_DIR is passed verbatim: a spec like '$SCRATCH/ref' must be
+        # expanded on the cluster, not by this shell.
+        args=(
+          --site "$SITE"
+          --remote-dir "$REMOTE_DIR"
+          --local-dir "$LOCAL_DIR"
+          --tar-dir "$tar_dir"
+        )
+        if [ -n "$TROIKA_CONFIG" ]; then args+=(--troika-config "$TROIKA_CONFIG"); fi
+        if [ -n "$TROIKA_USER" ]; then args+=(--troika-user "$TROIKA_USER"); fi
+        "$CI_INFRASTRUCTURE_PYTHON" -u -m ci_infrastructure.hpc fetch-tree "${args[@]}"