From 507310b605f9cf13b457227d3d2d55eb3bdc34d1 Mon Sep 17 00:00:00 2001 From: Oskar Weser Date: Thu, 30 Jul 2026 11:21:14 +0200 Subject: [PATCH 1/2] implemented fetch/push-hpc-tree commands including tests --- .github/workflows/hpc-transfer-e2e.yml | 95 ++++++++++++++++ HPC.md | 57 ++++++++++ actions/fetch-hpc-tree/action.yml | 78 +++++++++++++ actions/push-hpc-tree/action.yml | 77 +++++++++++++ src/ci_infrastructure/hpc/orchestrate.py | 70 ++++++++++++ src/ci_infrastructure/hpc/transfer.py | 86 +++++++++++++-- tests/test_hpc_orchestrate.py | 130 ++++++++++++++++++++++ tests/test_hpc_transfer.py | 133 +++++++++++++++++++++++ 8 files changed, 715 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/hpc-transfer-e2e.yml create mode 100644 actions/fetch-hpc-tree/action.yml create mode 100644 actions/push-hpc-tree/action.yml diff --git a/.github/workflows/hpc-transfer-e2e.yml b/.github/workflows/hpc-transfer-e2e.yml new file mode 100644 index 0000000..39f4c9e --- /dev/null +++ b/.github/workflows/hpc-transfer-e2e.yml @@ -0,0 +1,95 @@ +# SPDX-FileCopyrightText: 2026 European Centre for Medium-Range Weather Forecasts (ECMWF) +# +# SPDX-License-Identifier: Apache-2.0 + +name: HPC transfer e2e + +# Real-world usage test for the fetch-tree / push-tree primitives, with no +# cluster. It drives them against the local-direct troika site (connection: +# local), so troika runs the tar commands and getfile/sendfile on this runner — +# a genuine tar -> transfer -> untar round-trip with source and destination both +# local. Exercised twice: through the CLI, and through the fetch-hpc-tree / +# push-hpc-tree composite actions. +on: + push: + pull_request: + +jobs: + local-direct-roundtrip: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.12" + + - name: Install ci-infrastructure (this branch) + run: pip install -e . + + # The composite actions bootstrap via ensure-infrastructure-present@main, + # which would install ci-infrastructure from main — without the commands + # this branch adds. Point CI_INFRASTRUCTURE_PYTHON at the interpreter we + # just installed the branch into; ensure-infrastructure-present then takes + # its reuse fast path and the actions run this branch's code. + - name: Expose the branch interpreter to the composite actions + shell: bash + run: echo "CI_INFRASTRUCTURE_PYTHON=$(python -c 'import sys; print(sys.executable)')" >> "$GITHUB_ENV" + + - name: CLI round-trip through local-direct + shell: bash + run: | + set -euo pipefail + cfg=src/ci_infrastructure/hpc/troika-config.yml + src="$RUNNER_TEMP/cli/src" + mkdir -p "$src/nested" + echo runner-payload > "$src/hello.txt" + echo deep > "$src/nested/deep.txt" + + python -m ci_infrastructure.hpc push-tree --site local-direct --troika-config "$cfg" \ + --local-dir "$src" --remote-dir "$RUNNER_TEMP/cli/remote" --tar-dir "$RUNNER_TEMP/cli/tars" + python -m ci_infrastructure.hpc fetch-tree --site local-direct --troika-config "$cfg" \ + --remote-dir "$RUNNER_TEMP/cli/remote" --local-dir "$RUNNER_TEMP/cli/back" --tar-dir "$RUNNER_TEMP/cli/tars" + diff -r "$src" "$RUNNER_TEMP/cli/back" + + # A dry run resolves the path but must transfer nothing. + python -m ci_infrastructure.hpc fetch-tree --site local-direct --troika-config "$cfg" \ + --remote-dir "$RUNNER_TEMP/cli/remote" --local-dir "$RUNNER_TEMP/cli/dry" \ + --tar-dir "$RUNNER_TEMP/cli/tars" --dryrun + test ! -e "$RUNNER_TEMP/cli/dry" + echo "CLI round-trip OK" + + - name: Seed a tree for the action round-trip + shell: bash + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/act/src/nested" + echo action-payload > "$RUNNER_TEMP/act/src/hello.txt" + echo deep > "$RUNNER_TEMP/act/src/nested/deep.txt" + + - name: Push the tree up via the action + id: push + uses: ./actions/push-hpc-tree + with: + site: local-direct + local-dir: ${{ runner.temp }}/act/src + remote-dir: ${{ runner.temp }}/act/remote + + - name: Fetch the tree back via the action + id: fetch + uses: ./actions/fetch-hpc-tree + with: + site: local-direct + remote-dir: ${{ runner.temp }}/act/remote + local-dir: ${{ runner.temp }}/act/back + + - name: Assert the action outputs and the round-trip + shell: bash + env: + PUSH_REMOTE_DIR: ${{ steps.push.outputs.remote-dir }} + FETCH_LOCAL_DIR: ${{ steps.fetch.outputs.local-dir }} + run: | + set -euo pipefail + test "$PUSH_REMOTE_DIR" = "$RUNNER_TEMP/act/remote" + test "$FETCH_LOCAL_DIR" = "$RUNNER_TEMP/act/back" + diff -r "$RUNNER_TEMP/act/src" "$RUNNER_TEMP/act/back" + echo "Action round-trip OK" diff --git a/HPC.md b/HPC.md index 352d067..69d30f0 100644 --- a/HPC.md +++ b/HPC.md @@ -163,3 +163,60 @@ The submit-then-poll path leaves per-artifact `staging/`, `src/`, `install/` and on the login-node runner, sweeping per-artifact trees older than `N` days. Run it via `workflow_dispatch` with `dryrun: true` first to see what it would remove. + +## Moving extra directories between the runner and the cluster + +The build flow already brackets a job with two tree transfers over troika's +connection. The same transfer is also available standalone, for any workflow that +needs to move a directory in or out of the cluster outside a build — e.g. pulling +a job's reference/artifact directory back for a later processing step, or staging +inputs onto shared scratch before a job reads them. Both directions are a plain +login-node copy: **no scheduler and no S3**, so they work against `direct` sites +too. + +- **`fetch-tree`** — cluster → runner. Tars `--remote-dir` on the cluster, brings + the single tarball back and unpacks it into `--local-dir` on the runner. Writes + the runner-local directory as the `local-dir` output. +- **`push-tree`** — runner → cluster. Tars `--local-dir` on the runner, ships it up + and unpacks it into `--remote-dir` on the cluster. Writes the resolved cluster + directory as the `remote-dir` output. + +Two rules for the remote directory: + +- it must live on a filesystem the login node can reach (shared scratch — the same + Lustre `$SCRATCH` the login-node runner and the compute nodes all see); +- `--remote-dir` is expanded **on the cluster**, so quote a `$SCRATCH/…` spec to + keep the runner's shell from expanding it first (same rule as `--remote-work-dir` + — see *Why the work dir is expanded on the cluster, not on the runner*). + +As composite actions (post-step to pull a job's output back to the runner): + +```yaml +- uses: ecmwf/ci-infrastructure/actions/fetch-hpc-tree@main + with: + site: hpc-batch # same troika site the job used (or lumi) + troika-user: ${{ secrets.HPC_CI_SSH_USER }} + remote-dir: ${{ env.OUTPUT_DIR }}/ectrans-reference-artifact + local-dir: ./ectrans-reference-artifact +``` + +```yaml +- uses: ecmwf/ci-infrastructure/actions/push-hpc-tree@main + with: + site: hpc-batch + troika-user: ${{ secrets.HPC_CI_SSH_USER }} + local-dir: ./inputs + remote-dir: ${{ env.OUTPUT_DIR }}/inputs +``` + +Or directly, e.g. on the login-node runner: + +```bash +python -m ci_infrastructure.hpc fetch-tree --site hpc-batch \ + --remote-dir "$OUTPUT_DIR/ectrans-reference-artifact" \ + --local-dir ./ref --tar-dir "$RUNNER_TEMP/hpc-tars" +``` + +`.github/workflows/hpc-transfer-e2e.yml` exercises both commands and both actions +on every push against the `local-direct` site (a real tar → transfer → untar +round-trip with no cluster). diff --git a/actions/fetch-hpc-tree/action.yml b/actions/fetch-hpc-tree/action.yml new file mode 100644 index 0000000..58a5218 --- /dev/null +++ b/actions/fetch-hpc-tree/action.yml @@ -0,0 +1,78 @@ +# SPDX-FileCopyrightText: 2026 European Centre for Medium-Range Weather Forecasts (ECMWF) +# +# SPDX-License-Identifier: Apache-2.0 + +name: Fetch HPC tree +description: > + Copy a directory a job produced on the cluster back to the runner, over + troika's own connection (no scheduler, no S3). Use it as a post-step after an + HPC job to pull an output directory — reference data, artifacts — off shared + scratch and onto the runner for later processing. + + The remote directory must sit on a filesystem the login node can read (shared + scratch). It may name cluster variables such as SCRATCH; the spec is expanded + on the cluster, so pass it quoted so this runner does not expand it first. The + runner-local extraction directory is exposed as the local-dir output. + +inputs: + site: + description: 'Troika site name (see troika-config.yml), e.g. hpc-batch or lumi' + required: true + remote-dir: + description: 'Source directory on the cluster to fetch (may name cluster variables like $SCRATCH)' + required: true + local-dir: + description: 'Runner-local directory to unpack the tree into' + required: true + tar-dir: + description: 'Runner-local scratch dir for the transferred tarball. Empty falls back to $RUNNER_TEMP/hpc-tars.' + required: false + default: '' + troika-config: + description: 'Path to a troika config. Empty uses the one packaged with ci-infrastructure.' + required: false + default: '' + troika-user: + description: 'Remote/scheduler user for troika. Empty leaves troika to resolve it.' + required: false + default: '' + +outputs: + local-dir: + description: 'Runner-local directory the tree was unpacked into' + value: ${{ steps.fetch.outputs.local-dir }} + +runs: + using: composite + steps: + - uses: ecmwf/ci-infrastructure/actions/ensure-infrastructure-present@main + + - name: Fetch the tree off the cluster + id: fetch + shell: bash + env: + SITE: ${{ inputs.site }} + REMOTE_DIR: ${{ inputs.remote-dir }} + LOCAL_DIR: ${{ inputs.local-dir }} + TAR_DIR_INPUT: ${{ inputs.tar-dir }} + TROIKA_CONFIG: ${{ inputs.troika-config }} + TROIKA_USER: ${{ inputs.troika-user }} + run: | + set -euo pipefail + if [ -z "${CI_INFRASTRUCTURE_PYTHON:-}" ]; then + echo "::error::CI_INFRASTRUCTURE_PYTHON is unset — ensure-infrastructure-present did not run." >&2 + exit 1 + fi + tar_dir="${TAR_DIR_INPUT:-$RUNNER_TEMP/hpc-tars}" + mkdir -p "$tar_dir" + # REMOTE_DIR is passed verbatim: a spec like '$SCRATCH/ref' must be + # expanded on the cluster, not by this shell. + args=( + --site "$SITE" + --remote-dir "$REMOTE_DIR" + --local-dir "$LOCAL_DIR" + --tar-dir "$tar_dir" + ) + if [ -n "$TROIKA_CONFIG" ]; then args+=(--troika-config "$TROIKA_CONFIG"); fi + if [ -n "$TROIKA_USER" ]; then args+=(--troika-user "$TROIKA_USER"); fi + "$CI_INFRASTRUCTURE_PYTHON" -u -m ci_infrastructure.hpc fetch-tree "${args[@]}" diff --git a/actions/push-hpc-tree/action.yml b/actions/push-hpc-tree/action.yml new file mode 100644 index 0000000..3dc4dc8 --- /dev/null +++ b/actions/push-hpc-tree/action.yml @@ -0,0 +1,77 @@ +# SPDX-FileCopyrightText: 2026 European Centre for Medium-Range Weather Forecasts (ECMWF) +# +# SPDX-License-Identifier: Apache-2.0 + +name: Push HPC tree +description: > + Copy a runner-local directory up to a directory on the cluster, over troika's + own connection (no scheduler, no S3). Use it as a pre-step before an HPC job to + stage inputs onto shared scratch that the job then reads. + + The destination directory must sit on a filesystem the login node can write + (shared scratch). It may name cluster variables such as SCRATCH; the spec is + expanded on the cluster, so pass it quoted so this runner does not expand it + first. The resolved cluster directory is exposed as the remote-dir output. + +inputs: + site: + description: 'Troika site name (see troika-config.yml), e.g. hpc-batch or lumi' + required: true + local-dir: + description: 'Source directory on the runner to push' + required: true + remote-dir: + description: 'Destination directory on the cluster (may name cluster variables like $SCRATCH)' + required: true + tar-dir: + description: 'Runner-local scratch dir for the transferred tarball. Empty falls back to $RUNNER_TEMP/hpc-tars.' + required: false + default: '' + troika-config: + description: 'Path to a troika config. Empty uses the one packaged with ci-infrastructure.' + required: false + default: '' + troika-user: + description: 'Remote/scheduler user for troika. Empty leaves troika to resolve it.' + required: false + default: '' + +outputs: + remote-dir: + description: 'Resolved cluster directory the tree was unpacked into' + value: ${{ steps.push.outputs.remote-dir }} + +runs: + using: composite + steps: + - uses: ecmwf/ci-infrastructure/actions/ensure-infrastructure-present@main + + - name: Push the tree up to the cluster + id: push + shell: bash + env: + SITE: ${{ inputs.site }} + LOCAL_DIR: ${{ inputs.local-dir }} + REMOTE_DIR: ${{ inputs.remote-dir }} + TAR_DIR_INPUT: ${{ inputs.tar-dir }} + TROIKA_CONFIG: ${{ inputs.troika-config }} + TROIKA_USER: ${{ inputs.troika-user }} + run: | + set -euo pipefail + if [ -z "${CI_INFRASTRUCTURE_PYTHON:-}" ]; then + echo "::error::CI_INFRASTRUCTURE_PYTHON is unset — ensure-infrastructure-present did not run." >&2 + exit 1 + fi + tar_dir="${TAR_DIR_INPUT:-$RUNNER_TEMP/hpc-tars}" + mkdir -p "$tar_dir" + # REMOTE_DIR is passed verbatim: a spec like '$SCRATCH/inputs' must be + # expanded on the cluster, not by this shell. + args=( + --site "$SITE" + --local-dir "$LOCAL_DIR" + --remote-dir "$REMOTE_DIR" + --tar-dir "$tar_dir" + ) + if [ -n "$TROIKA_CONFIG" ]; then args+=(--troika-config "$TROIKA_CONFIG"); fi + if [ -n "$TROIKA_USER" ]; then args+=(--troika-user "$TROIKA_USER"); fi + "$CI_INFRASTRUCTURE_PYTHON" -u -m ci_infrastructure.hpc push-tree "${args[@]}" diff --git a/src/ci_infrastructure/hpc/orchestrate.py b/src/ci_infrastructure/hpc/orchestrate.py index 8b685df..2080e7e 100644 --- a/src/ci_infrastructure/hpc/orchestrate.py +++ b/src/ci_infrastructure/hpc/orchestrate.py @@ -731,5 +731,75 @@ def gc( run_gc(site._connection, remote_work_dir=resolved, older_than_days=older_than_days, dryrun=dryrun) +@main.command("fetch-tree", help="Copy a directory a job produced off the cluster back to the runner.") +@click.option("--site", "site_name", required=True, help="Troika site name") +@click.option("--troika-config", "troika_config", default=None, help="Path to troika config (default: packaged)") +@click.option("--troika-user", "troika_user", default=None, help="Remote/scheduler user for troika") +@click.option( + "--remote-dir", + "remote_dir", + required=True, + help="Source directory on the cluster to fetch. May name cluster variables (e.g. '$SCRATCH/ref'); " + "expanded on the cluster, not on the runner.", +) +@click.option("--local-dir", "local_dir", required=True, help="Runner-local directory to unpack the tree into") +@click.option("--tar-dir", "tar_dir", required=True, help="Runner-local scratch dir for the transferred tarball") +@click.option("--dryrun", is_flag=True, default=False, help="Resolve the remote path but transfer nothing") +def fetch_tree_cmd( + site_name: str, + troika_config: str | None, + troika_user: str | None, + remote_dir: str, + local_dir: str, + tar_dir: str, + dryrun: bool, +) -> None: + site = load_site(site_name, config_path=troika_config, user=troika_user) + resolved = resolve_remote_path(site._connection, remote_dir) # the source lives on the cluster + if resolved != remote_dir: + print(f"fetch-tree: remote dir {remote_dir!r} -> {resolved}") + transfer.fetch_tree(site._connection, remote_dir=resolved, local_dir=local_dir, tar_dir=tar_dir, dryrun=dryrun) + if dryrun: + print(f"fetch-tree: dry run; would fetch {resolved} -> {local_dir}") + return + _write_output("local-dir", local_dir) + print(f"fetch-tree: fetched {resolved} -> {local_dir}") + + +@main.command("push-tree", help="Copy a runner-local directory up to a directory on the cluster.") +@click.option("--site", "site_name", required=True, help="Troika site name") +@click.option("--troika-config", "troika_config", default=None, help="Path to troika config (default: packaged)") +@click.option("--troika-user", "troika_user", default=None, help="Remote/scheduler user for troika") +@click.option("--local-dir", "local_dir", required=True, help="Source directory on the runner to push") +@click.option( + "--remote-dir", + "remote_dir", + required=True, + help="Destination directory on the cluster. May name cluster variables (e.g. '$SCRATCH/inputs'); " + "expanded on the cluster, not on the runner.", +) +@click.option("--tar-dir", "tar_dir", required=True, help="Runner-local scratch dir for the transferred tarball") +@click.option("--dryrun", is_flag=True, default=False, help="Resolve the remote path but transfer nothing") +def push_tree_cmd( + site_name: str, + troika_config: str | None, + troika_user: str | None, + local_dir: str, + remote_dir: str, + tar_dir: str, + dryrun: bool, +) -> None: + site = load_site(site_name, config_path=troika_config, user=troika_user) + resolved = resolve_remote_path(site._connection, remote_dir) # the destination lives on the cluster + if resolved != remote_dir: + print(f"push-tree: remote dir {remote_dir!r} -> {resolved}") + transfer.push_tree(site._connection, local_dir=local_dir, remote_dir=resolved, tar_dir=tar_dir, dryrun=dryrun) + if dryrun: + print(f"push-tree: dry run; would push {local_dir} -> {resolved}") + return + _write_output("remote-dir", resolved) + print(f"push-tree: pushed {local_dir} -> {resolved}") + + if __name__ == "__main__": main() diff --git a/src/ci_infrastructure/hpc/transfer.py b/src/ci_infrastructure/hpc/transfer.py index 30cd1d1..2332a68 100644 --- a/src/ci_infrastructure/hpc/transfer.py +++ b/src/ci_infrastructure/hpc/transfer.py @@ -171,27 +171,91 @@ def _ship_prefix(conn: Connection, *, local_prefix: str, remote_dir: str, tar_pa ) -def fetch_install( +def fetch_tree( conn: Connection, *, - remote_install_dir: str, - local_install_dir: str, + remote_dir: str, + local_dir: str, tar_dir: str, + tarball_suffix: str = "fetch", dryrun: bool = False, ) -> None: - """Tar the cluster's install tree and unpack it into ``local_install_dir`` on the runner.""" + """Tar a directory on the cluster and unpack it into ``local_dir`` on the runner. + + The generic HPC->runner half of the two bracketing transfers, driven straight + over troika's connection (no scheduler, so it works against ``direct`` sites + too). The tree moves as a single ``..tgz`` next to the + source on the cluster, because troika's connection transfers one file at a + time; tar/untar on each side turns that into a directory copy. + """ if dryrun: return - remote = PurePosixPath(remote_install_dir) - remote_tgz = str(remote.parent / f"{remote.name}.install.tgz") - local_tgz = Path(tar_dir) / f"{remote.name}.install.tgz" + remote = PurePosixPath(remote_dir) + remote_tgz = str(remote.parent / f"{remote.name}.{tarball_suffix}.tgz") + local_tgz = Path(tar_dir) / f"{remote.name}.{tarball_suffix}.tgz" _run_remote( conn, - ["bash", "-c", f"tar -czf {shlex.quote(remote_tgz)} -C {shlex.quote(remote_install_dir)} ."], - what="Remote install tar", + ["bash", "-c", f"tar -czf {shlex.quote(remote_tgz)} -C {shlex.quote(remote_dir)} ."], + what="Remote tree tar", ) Path(tar_dir).mkdir(parents=True, exist_ok=True) conn.getfile(remote_tgz, local_tgz) - Path(local_install_dir).mkdir(parents=True, exist_ok=True) - subprocess.run(["tar", "-xzf", str(local_tgz), "-C", str(local_install_dir)], check=True) + Path(local_dir).mkdir(parents=True, exist_ok=True) + subprocess.run(["tar", "-xzf", str(local_tgz), "-C", str(local_dir)], check=True) + + +def push_tree( + conn: Connection, + *, + local_dir: str, + remote_dir: str, + tar_dir: str, + tarball_suffix: str = "push", + dryrun: bool = False, +) -> None: + """Tar ``local_dir`` on the runner and unpack it into ``remote_dir`` on the cluster. + + The generic runner->HPC mirror of :func:`fetch_tree`: local tar -> ``sendfile`` + -> remote ``mkdir -p`` + untar. Like ``fetch_tree`` it needs no scheduler, so + it works against ``direct`` sites too. (This is the same shape as the private + ``_ship_prefix``, which stays tied to the source-shipping flow.) + """ + if dryrun: + return + name = PurePosixPath(remote_dir).name + local_tgz = Path(tar_dir) / f"{name}.{tarball_suffix}.tgz" + remote_tgz = f"{remote_dir.rstrip('/')}.{tarball_suffix}.tgz" + + Path(tar_dir).mkdir(parents=True, exist_ok=True) + subprocess.run(["tar", "-czf", str(local_tgz), "-C", str(local_dir), "."], check=True) + _run_remote(conn, ["mkdir", "-p", remote_dir], what=f"Remote mkdir of {remote_dir}") + conn.sendfile(local_tgz, remote_tgz) + _run_remote( + conn, + ["bash", "-c", f"tar -xzf {shlex.quote(remote_tgz)} -C {shlex.quote(remote_dir)}"], + what="Remote tree unpack", + ) + + +def fetch_install( + conn: Connection, + *, + remote_install_dir: str, + local_install_dir: str, + tar_dir: str, + dryrun: bool = False, +) -> None: + """Tar the cluster's install tree and unpack it into ``local_install_dir`` on the runner. + + Thin wrapper over :func:`fetch_tree` that keeps the build flow's + ``.install.tgz`` temp-tarball name. + """ + fetch_tree( + conn, + remote_dir=remote_install_dir, + local_dir=local_install_dir, + tar_dir=tar_dir, + tarball_suffix="install", + dryrun=dryrun, + ) diff --git a/tests/test_hpc_orchestrate.py b/tests/test_hpc_orchestrate.py index d8506a1..4252259 100644 --- a/tests/test_hpc_orchestrate.py +++ b/tests/test_hpc_orchestrate.py @@ -754,5 +754,135 @@ def test_submit_wait_publish_mode_checks_cache_and_fetches(monkeypatch: pytest.M assert calls == {"object_exists": 1, "fetch_install": 1} +# --------------------------------------------------------------------------- # +# fetch-tree / push-tree (the standalone transfer subcommands) +# +# Driven through the CLI with load_site / resolve_remote_path / the transfer +# primitive stubbed, so we assert the two things the command wires: the kwargs +# forwarded to the primitive, and the $GITHUB_OUTPUT it writes. +# --------------------------------------------------------------------------- # +class _FakeSite: + def __init__(self) -> None: + self._connection = RecordingConnection() + + +def _stub_site_and_resolver(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(orch, "load_site", lambda *a, **k: _FakeSite()) + monkeypatch.setattr(orch, "resolve_remote_path", lambda _conn, spec: spec) # identity + + +def test_fetch_tree_forwards_kwargs_and_writes_output(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + _stub_site_and_resolver(monkeypatch) + captured: dict[str, object] = {} + monkeypatch.setattr(transfer, "fetch_tree", lambda _conn, **kw: captured.update(kw)) + out_file = tmp_path / "gh_output" + monkeypatch.setenv("GITHUB_OUTPUT", str(out_file)) + + result = CliRunner().invoke( + orch.fetch_tree_cmd, + [ + "--site", + "local-direct", + "--remote-dir", + "/scratch/ref", + "--local-dir", + str(tmp_path / "back"), + "--tar-dir", + str(tmp_path / "tars"), + ], + ) + assert result.exit_code == 0, result.output + assert captured == { + "remote_dir": "/scratch/ref", + "local_dir": str(tmp_path / "back"), + "tar_dir": str(tmp_path / "tars"), + "dryrun": False, + } + assert out_file.read_text() == f"local-dir={tmp_path / 'back'}\n" + + +def test_fetch_tree_dryrun_forwards_flag_and_writes_no_output(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + _stub_site_and_resolver(monkeypatch) + captured: dict[str, object] = {} + monkeypatch.setattr(transfer, "fetch_tree", lambda _conn, **kw: captured.update(kw)) + out_file = tmp_path / "gh_output" + monkeypatch.setenv("GITHUB_OUTPUT", str(out_file)) + + result = CliRunner().invoke( + orch.fetch_tree_cmd, + [ + "--site", + "local-direct", + "--remote-dir", + "/scratch/ref", + "--local-dir", + str(tmp_path / "back"), + "--tar-dir", + str(tmp_path / "tars"), + "--dryrun", + ], + ) + assert result.exit_code == 0, result.output + assert captured["dryrun"] is True + assert not out_file.exists() # nothing written on a dry run + + +def test_push_tree_forwards_kwargs_and_writes_output(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + _stub_site_and_resolver(monkeypatch) + captured: dict[str, object] = {} + monkeypatch.setattr(transfer, "push_tree", lambda _conn, **kw: captured.update(kw)) + out_file = tmp_path / "gh_output" + monkeypatch.setenv("GITHUB_OUTPUT", str(out_file)) + + result = CliRunner().invoke( + orch.push_tree_cmd, + [ + "--site", + "local-direct", + "--local-dir", + str(tmp_path / "inputs"), + "--remote-dir", + "/scratch/inputs", + "--tar-dir", + str(tmp_path / "tars"), + ], + ) + assert result.exit_code == 0, result.output + assert captured == { + "local_dir": str(tmp_path / "inputs"), + "remote_dir": "/scratch/inputs", + "tar_dir": str(tmp_path / "tars"), + "dryrun": False, + } + # push writes the RESOLVED cluster dir (its useful output for later steps). + assert out_file.read_text() == "remote-dir=/scratch/inputs\n" + + +def test_push_tree_dryrun_forwards_flag_and_writes_no_output(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + _stub_site_and_resolver(monkeypatch) + captured: dict[str, object] = {} + monkeypatch.setattr(transfer, "push_tree", lambda _conn, **kw: captured.update(kw)) + out_file = tmp_path / "gh_output" + monkeypatch.setenv("GITHUB_OUTPUT", str(out_file)) + + result = CliRunner().invoke( + orch.push_tree_cmd, + [ + "--site", + "local-direct", + "--local-dir", + str(tmp_path / "inputs"), + "--remote-dir", + "/scratch/inputs", + "--tar-dir", + str(tmp_path / "tars"), + "--dryrun", + ], + ) + assert result.exit_code == 0, result.output + assert captured["dryrun"] is True + assert not out_file.exists() + + if __name__ == "__main__": # pragma: no cover raise SystemExit(pytest.main([__file__, "-v"])) diff --git a/tests/test_hpc_transfer.py b/tests/test_hpc_transfer.py index 149dc40..083ec83 100644 --- a/tests/test_hpc_transfer.py +++ b/tests/test_hpc_transfer.py @@ -225,6 +225,139 @@ def getfile(self, src: Any, dst: Any, dryrun: bool = False) -> None: assert (tmp_path / "local-install").is_dir() +# --------------------------------------------------------------------------- # +# fetch_tree / push_tree (the generic HPC<->runner primitives) +# --------------------------------------------------------------------------- # +def test_fetch_tree_tar_getfile_unpack_order(tmp_path: Path) -> None: + class TarballConnection(FakeConnection): + # fetch_tree really untars what getfile delivered, so hand it a valid + # (empty) tarball rather than a no-op. + def getfile(self, src: Any, dst: Any, dryrun: bool = False) -> None: + super().getfile(src, dst) + with tarfile.open(dst, "w:gz"): + pass + + conn = TarballConnection() + transfer.fetch_tree( + conn, + remote_dir="/remote/ref/art", + local_dir=str(tmp_path / "local"), + tar_dir=str(tmp_path / "stage"), + ) + # Remote tar first, then getfile back under the default .fetch.tgz name. + assert conn.executed[0][:2] == ["bash", "-c"] + assert "tar -czf" in conn.executed[0][2] and "/remote/ref/art" in conn.executed[0][2] + assert conn.fetched == [("/remote/ref/art.fetch.tgz", str(tmp_path / "stage" / "art.fetch.tgz"))] + assert (tmp_path / "local").is_dir() + + +def test_fetch_tree_dryrun_does_nothing(tmp_path: Path) -> None: + conn = FakeConnection() + transfer.fetch_tree( + conn, + remote_dir="/remote/ref/art", + local_dir=str(tmp_path / "local"), + tar_dir=str(tmp_path / "stage"), + dryrun=True, + ) + assert conn.executed == [] and conn.fetched == [] + assert not (tmp_path / "local").exists() + + +def test_fetch_install_still_names_install_tgz(tmp_path: Path) -> None: + """Regression guard: the wrapper keeps the build flow's .install.tgz name.""" + + class TarballConnection(FakeConnection): + def getfile(self, src: Any, dst: Any, dryrun: bool = False) -> None: + super().getfile(src, dst) + with tarfile.open(dst, "w:gz"): + pass + + conn = TarballConnection() + transfer.fetch_install( + conn, + remote_install_dir="/remote/install/art", + local_install_dir=str(tmp_path / "local-install"), + tar_dir=str(tmp_path / "stage"), + ) + assert conn.fetched == [("/remote/install/art.install.tgz", str(tmp_path / "stage" / "art.install.tgz"))] + + +def test_push_tree_tar_sendfile_unpack_order(tmp_path: Path) -> None: + src = _make_tree(tmp_path / "inputs", "in.txt", "payload") + conn = FakeConnection() + transfer.push_tree( + conn, + local_dir=str(src), + remote_dir="/remote/inputs/art", + tar_dir=str(tmp_path / "stage"), + ) + # Remote dir is created, then the tarball is scp'd up under .push.tgz and untarred. + assert ["mkdir", "-p", "/remote/inputs/art"] in conn.executed + assert conn.sent == [(str(tmp_path / "stage" / "art.push.tgz"), "/remote/inputs/art.push.tgz")] + untars = [c[2] for c in conn.executed if c[:2] == ["bash", "-c"]] + assert any("tar -xzf" in u and "/remote/inputs/art" in u for u in untars) + + +def test_push_tree_dryrun_does_nothing(tmp_path: Path) -> None: + src = _make_tree(tmp_path / "inputs", "in.txt", "payload") + conn = FakeConnection() + transfer.push_tree( + conn, + local_dir=str(src), + remote_dir="/remote/inputs/art", + tar_dir=str(tmp_path / "stage"), + dryrun=True, + ) + assert conn.executed == [] and conn.sent == [] + assert not (tmp_path / "stage").exists() # no local tarball either + + +def test_push_then_fetch_roundtrip_preserves_tree(tmp_path: Path) -> None: + """push_tree stages a tree on the 'cluster'; fetch_tree brings it back intact.""" + src = _make_tree(tmp_path / "inputs", "hello.txt", "content-xyz") + + class CopyingConnection(FakeConnection): + # Emulate scp by really copying the file, and run the remote commands + # locally so the round-trip actually moves bytes. + def sendfile(self, src: Any, dst: Any, dryrun: bool = False) -> None: + super().sendfile(src, dst) + Path(dst).parent.mkdir(parents=True, exist_ok=True) + Path(dst).write_bytes(Path(src).read_bytes()) + + def getfile(self, src: Any, dst: Any, dryrun: bool = False) -> None: + super().getfile(src, dst) + Path(dst).write_bytes(Path(src).read_bytes()) + + def execute(self, command: Any, stdout: Any = None, stderr: Any = None, dryrun: bool = False) -> FakeProc: + proc = super().execute(command, stdout, stderr, dryrun) + argv = [str(c) for c in command] + if argv[:2] == ["bash", "-c"]: + subprocess.run(argv[2], shell=True, check=True) + else: + subprocess.run(argv, check=True) # mkdir -p + return proc + + conn = CopyingConnection() + remote = tmp_path / "remote" / "inputs" / "art" + transfer.push_tree( + conn, + local_dir=str(src), + remote_dir=str(remote), + tar_dir=str(tmp_path / "stage"), + ) + assert (remote / "hello.txt").read_text() == "content-xyz" + + fetched = tmp_path / "back" + transfer.fetch_tree( + conn, + remote_dir=str(remote), + local_dir=str(fetched), + tar_dir=str(tmp_path / "stage2"), + ) + assert (fetched / "hello.txt").read_text() == "content-xyz" + + def test_ship_then_fetch_roundtrip_preserves_tree(tmp_path: Path) -> None: """The tarball ship stages unpacks intact, and fetch_install brings a tree back.""" src = _make_tree(tmp_path / "checkout", "hello.txt", "content-xyz") From ec9d67e07e890c055e319c48a33a93b95b11d3f6 Mon Sep 17 00:00:00 2001 From: Oskar Weser Date: Thu, 30 Jul 2026 12:49:45 +0200 Subject: [PATCH 2/2] added actual HPC roundtrip --- .github/workflows/ci.yml | 3 + .github/workflows/hpc-transfer-e2e.yml | 107 +++++++++++++++++++++++-- 2 files changed, 104 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 83b15c7..4429d68 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,7 +5,10 @@ name: CI on: + # push scoped to main so a branch with an open PR runs the pull_request copy + # only, not a redundant second push run. push: + branches: [main] pull_request: jobs: diff --git a/.github/workflows/hpc-transfer-e2e.yml b/.github/workflows/hpc-transfer-e2e.yml index 39f4c9e..2054af9 100644 --- a/.github/workflows/hpc-transfer-e2e.yml +++ b/.github/workflows/hpc-transfer-e2e.yml @@ -4,15 +4,24 @@ name: HPC transfer e2e -# Real-world usage test for the fetch-tree / push-tree primitives, with no -# cluster. It drives them against the local-direct troika site (connection: -# local), so troika runs the tar commands and getfile/sendfile on this runner — -# a genuine tar -> transfer -> untar round-trip with source and destination both -# local. Exercised twice: through the CLI, and through the fetch-hpc-tree / -# push-hpc-tree composite actions. +# Real-world usage test for the fetch-tree / push-tree primitives, in two tiers: +# +# * local-direct-roundtrip (every push/PR, ubuntu-latest, no cluster) drives the +# commands against the local-direct troika site (connection: local), so troika +# runs the tar commands and getfile/sendfile on this runner — a genuine +# tar -> transfer -> untar round-trip with source and destination both local. +# Exercised through the CLI and through the composite actions. +# * hpc-roundtrip (manual only, self-hosted [hpc] login-node runner) does the +# real thing: pushes a tree to shared scratch on the cluster over troika ssh +# and fetches it back. +# +# push is scoped to main so a branch with an open PR runs the pull_request copy +# only, not a redundant second push run. on: push: + branches: [main] pull_request: + workflow_dispatch: jobs: local-direct-roundtrip: @@ -93,3 +102,89 @@ jobs: test "$FETCH_LOCAL_DIR" = "$RUNNER_TEMP/act/back" diff -r "$RUNNER_TEMP/act/src" "$RUNNER_TEMP/act/back" echo "Action round-trip OK" + + hpc-roundtrip: + # Manual only: it reaches the real cluster over troika ssh from the login-node + # self-hosted runner. The ubuntu-latest job above is the per-push gate. + if: github.event_name == 'workflow_dispatch' + runs-on: [hpc] + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + + - uses: ecmwf/ci-infrastructure/actions/ensure-infrastructure-present@main + + - name: Install this checkout's ci-infrastructure into a test venv + shell: bash + run: | + set -euo pipefail + # ensure-infrastructure-present bootstraps a known-good interpreter, but it + # installs ci-infrastructure from @main — which pre-merge lacks fetch-tree / + # push-tree. Build a venv from THIS checkout so the test runs the checked-out + # code, and point CI_INFRASTRUCTURE_PYTHON at it; the composite actions below + # then take their reuse fast path and run this code. + if [ -z "${CI_INFRASTRUCTURE_PYTHON:-}" ]; then + echo "::error::CI_INFRASTRUCTURE_PYTHON is unset — ensure-infrastructure-present did not run." >&2 + exit 1 + fi + venv="$RUNNER_TEMP/transfer-e2e-venv" + rm -rf "$venv" + "$CI_INFRASTRUCTURE_PYTHON" -m venv "$venv" + "$venv/bin/pip" install --disable-pip-version-check -e . + echo "CI_INFRASTRUCTURE_PYTHON=$venv/bin/python" >> "$GITHUB_ENV" + + - name: Seed a tree to push to the cluster + shell: bash + run: | + set -euo pipefail + src="$RUNNER_TEMP/hpc-e2e/src" + mkdir -p "$src/nested" + echo "hpc-e2e $GITHUB_RUN_ID" > "$src/hello.txt" + echo deep > "$src/nested/deep.txt" + + - name: Push the tree up to the cluster + id: push + uses: ./actions/push-hpc-tree + with: + site: hpc-batch + troika-user: ${{ secrets.HPC_CI_SSH_USER }} + local-dir: ${{ runner.temp }}/hpc-e2e/src + remote-dir: ${{ vars.HPC_CI_REMOTE_WORK_DIR }}/transfer-e2e-${{ github.run_id }} + + - name: Fetch the tree back from the cluster + id: fetch + uses: ./actions/fetch-hpc-tree + with: + site: hpc-batch + troika-user: ${{ secrets.HPC_CI_SSH_USER }} + remote-dir: ${{ steps.push.outputs.remote-dir }} + local-dir: ${{ runner.temp }}/hpc-e2e/back + + - name: Assert the round-trip + shell: bash + run: | + set -euo pipefail + diff -r "$RUNNER_TEMP/hpc-e2e/src" "$RUNNER_TEMP/hpc-e2e/back" + echo "HPC round-trip OK" + + - name: Remove the cluster scratch dir + if: always() + shell: bash + env: + TROIKA_USER: ${{ secrets.HPC_CI_SSH_USER }} + REMOTE_DIR: ${{ steps.push.outputs.remote-dir }} + run: | + set -euo pipefail + case "${REMOTE_DIR:-}" in + /*) ;; # only proceed for an absolute path the push step resolved + *) echo "nothing to clean (push did not resolve a remote dir)"; exit 0 ;; + esac + [ -n "${CI_INFRASTRUCTURE_PYTHON:-}" ] || exit 0 + # Every python line sits at this block's indent so the YAML block scalar + # keeps them; after the indent is stripped they are column-0 statements + # (a one-line comprehension, so there is no indented loop body to break it). + "$CI_INFRASTRUCTURE_PYTHON" -c 'import os, subprocess + from ci_infrastructure.hpc.site import load_site + conn = load_site("hpc-batch", user=(os.environ.get("TROIKA_USER") or None))._connection + base = os.environ["REMOTE_DIR"] + [conn.execute(["rm", "-rf", p], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() for p in (base, base + ".push.tgz", base + ".fetch.tgz")] + print("cleaned", base)'