remove oras

.github/workflows/build_and_test.yml

@@ -52,11 +52,24 @@ jobs:
         run: sudo ./meson/meson install -Cbuild --destdir=$PWD/staging --no-rebuild --skip-subprojects
       - name: create mount points for tests
         run: sudo mkdir /user-tools /user-environment
+      - name: verify zot is available for registry tests
+        # the zot binary is fetched at build time by test/integration/install-zot,
+        # which tolerates a failed download so local/offline builds still work (the
+        # registry suite then self-skips). in CI we require it, so live OCI
+        # round-trip coverage cannot silently vanish on a download hiccup.
+        run: |
+          test -x build/test/zot || {
+            echo "::error::zot binary missing (build/test/zot) - registry tests would silently skip" >&2
+            exit 1
+          }
       - name: unit tests
         run: ./meson/meson test -Cbuild --verbose unit
       - name: cli integration tests
         run: |
           UENV_BATS_SKIP_START=on STAGING_PATH=$PWD/staging/usr/local ./meson/meson test -Cbuild --verbose cli
+      - name: registry integration tests
+        run: |
+          UENV_BATS_SKIP_START=on STAGING_PATH=$PWD/staging/usr/local ./meson/meson test -Cbuild --verbose registry
       - name: squashfs-mount integration tests
         run: |
           UENV_BATS_SKIP_START=on STAGING_PATH=$PWD/staging/usr/local ./meson/meson test -Cbuild --verbose squashfs-mount

.gitignore

@@ -12,7 +12,9 @@ subprojects/spdlog-*
 subprojects/sqlite-amalgamation-*
 subprojects/nlohmann_json-*
 subprojects/tomlplusplus-*
+subprojects/zlib-*
 subprojects/.wraplock
+subprojects/.wrapdb
 
 # logs and artifacts generated by packaging workflows
 packaging/*.log

AGENTS.md

@@ -0,0 +1 @@
+CLAUDE.md

CLAUDE.md

@@ -1,6 +1,8 @@
 # CLAUDE.md
 
-This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+This file provides guidance to Claude Code (claude.ai/code) and other coding
+agents when working with code in this repository. It is the single source of
+truth; `AGENTS.md` is a symlink to this file.
 
 ## Project Overview
 
@@ -40,15 +42,15 @@ Configure via `-Doption=value` with `meson setup`:
 - `cli=true|false` - Build CLI tool (default: true)
 - `slurm_plugin=true|false` - Build Slurm plugin (default: true)
 - `squashfs_mount=true|false` - Build squashfs-mount helper (default: false)
-- `oras_version=X.Y.Z` - ORAS version to download (default: 1.2.0)
 
 ## Testing
 
-Four test suites exist:
+Five test suites exist:
 1. **unit** - C++ unit tests using Catch2 (in `test/unit/`)
 2. **cli** - CLI integration tests using BATS (in `test/integration/cli.bats`)
 3. **slurm** - Slurm plugin tests using BATS (in `test/integration/slurm.bats`)
 4. **squashfs-mount** - setuid helper tests using BATS (in `test/integration/squashfs-mount.bats`)
+5. **registry** - `uenv push`/`pull` against a throwaway local zot registry (in `test/integration/registry.bats`); self-skips when no zot binary is available
 
 ### Running Tests
 
@@ -57,10 +59,30 @@ To run the tests, run the tests directly instead of running them through meson.
 ```bash
 # Run tests directly
 ./test/unit                      # Unit tests
-./test/bats ./test/cli.bats     # CLI tests
-./test/bats ./test/slurm.bats   # Slurm tests
+./test/bats ./test/cli.bats      # CLI tests
+./test/bats ./test/slurm.bats    # Slurm tests
+./test/bats ./test/registry.bats # Registry (push/pull) tests
 ```
 
+### System name in tests
+
+Most CLI/Slurm tests resolve uenvs by a bare label (`app/42.0`, `tool`) and rely
+on the default system being `arapiles` (the repo records are stored `@arapiles`).
+
+The default system name comes from the config layers, merged in this order (later
+wins): `CLUSTER_NAME` env var → system config (`/etc/uenv/config.toml`) → user
+config (`$XDG_CONFIG_HOME/uenv/config.toml` or `$HOME/.config/uenv/config.toml`) →
+the `--system` CLI flag. On Alps the deployed system config sets
+`system_name = 'eiger'`, which **overrides `CLUSTER_NAME=arapiles`**. So exporting
+`CLUSTER_NAME` in a test is not sufficient.
+
+To force the system name, `cli.bats` and `slurm.bats` `setup()` write a throwaway
+user config that sets `system_name = 'arapiles'` and point `XDG_CONFIG_HOME` at it
+(user config beats system config). A test that writes its own `config.toml` must
+include `system_name = 'arapiles'` (or append to the file created in `setup()`
+with `>>` rather than clobbering it with `>`). Alternatively, pass `--system` on
+the command line.
+
 ### Elastic mock (`elastic_mock`)
 
 The slurm tests include an elastic telemetry test that uses a standalone mock server at `test/integration/elastic_mock`. The meson build copies it to `$BUILD_PATH/test/elastic_mock` (with execute permissions), and `setup_suite.bash` adds `$BUILD_PATH/test` to `PATH` so it is available by name in all BATS tests.
@@ -86,6 +108,30 @@ elastic_mock kill /tmp/cap.json
 
 In BATS tests the helper functions in `common.bash` wrap the common lifecycle: `start_elastic_mock CAPTURE_FILE PORT` (backgrounds `serve` and calls `wait-server`), `stop_elastic_mock` (kills by PID), and `wait_elastic_post CAPTURE_FILE [TIMEOUT]`.
 
+### Registry mock (`registry_ctl`, `listing_mock`)
+
+The `registry` suite exercises the native OCI client (`src/oci`) end-to-end against
+a real registry, without containers or the old `oras` binary. Two helper scripts
+are built into `$BUILD_PATH/test` (and so are on `PATH` in all BATS tests):
+
+- `registry_ctl` — manages the lifecycle of a throwaway [zot](https://zotregistry.dev)
+  registry (a single static binary). Subcommands include `runtime` (report
+  the zot binary, empty if unavailable → suite self-skips), `free-port`, `serve
+  STATE PORT` (backgrounds itself), `wait PORT --timeout N`, `digest PORT REPO REF`,
+  and `kill STATE`.
+  The zot binary is fetched at build time by `test/integration/install-zot`.
+- `listing_mock` — stands in for the CSCS uenv listing service
+  (`https://uenv-list.svc.cscs.ch/list`); same `free-port`/`serve`/`wait-server`/`kill`
+  lifecycle. Point uenv at it with `registry.listing_url` in the config.
+
+`registry.bats` drives `registry_ctl serve` / `listing_mock serve` directly from
+its `setup_file` (the servers must outlive individual tests, so their state is
+exported rather than held in `common.bash` helper vars). It writes a user
+`config.toml` whose `[registry]` block sets `url`, `default_namespace`, and
+`listing_url` to point at the local zot + listing_mock. The `[registry]` unit tests
+in `test/unit/oci_registry.cpp` cover the OCI round-trip directly by starting their
+own zot.
+
 ### Testing squashfs-mount
 
 The `squashfs-mount` helper requires setuid installation to test:
@@ -120,11 +166,11 @@ sudo meson install --destdir=$STAGE --no-rebuild --skip-subprojects
   - Parsing (`parse.h/cpp`, `lex.h` in util)
   - Mounting (`mount.h/cpp`)
   - Meta data (`meta.h/cpp`)
-  - Container registry interaction (`oras.h/cpp`)
   - Views (`view.h/cpp`)
   - Telemetry (`telemetry.h/cpp`, `elastic.h/cpp`)
   - Logging (`log.h/cpp`, `print.h/cpp`)
   - Settings management (`settings.h/cpp`)
+- `src/oci/` - Native OCI registry client (container registry interaction: pull, push, copy, manifests, auth); replaces the external `oras` binary. See "Self-contained `src/oci`" below.
 - `src/util/` - Utility libraries (color, curl, envvars, fs, lex, lustre, semver, shell, signal, strings, subprocess, toml)
 - `src/site/` - Site-specific configuration (CSCS-specific logic)
 - `src/slurm/` - Slurm plugin implementation
@@ -166,12 +212,10 @@ All dependencies are built as static libraries via meson wrap:
 - nlohmann_json - JSON parsing
 - sqlite3 - database
 - libcurl - HTTP operations
+- zlib - gzip handling in the native OCI registry client (`src/oci`)
 - Catch2 - testing (when tests enabled)
 - barkeep - progress indicators (header-only in `extern/`)
 
-The CLI build also downloads the ORAS binary for OCI registry operations.
-ORAS is installed in `prefix/libexec` for runtime use by the CLI.
-
 ## Development Notes
 
 ### Code Style
@@ -185,14 +229,45 @@ Use `util::expected<T, E>` (similar to std::expected) for fallible operations. T
 ### Parsing
 
 The codebase has a custom lexer in `src/util/lex.h/cpp` for tokenizing inputs.
-Parse functions are in `src/uenv/parse.h/cpp` and return `util::expected<T, parse_error>`.
+The shared parsing scaffolding — the `parse_error` type, the `PARSE` macro, and
+the `parse_string` primitive — lives in `src/util/parse.h/cpp` (namespace `util`)
+so that it can be reused by any module.
+Parse functions return `util::expected<T, util::parse_error>`. The `uenv` parsers
+are in `src/uenv/parse.h/cpp`; the OCI client has its own parsers in
+`src/oci/parse.h/cpp` (see "Self-contained `src/oci`" below).
 All inputs are parsed instead of using regex or simple string processing.
 When we have to parse a new input type:
 1. update the lexer (if needed)
-2. add a `parse` function in `src/uenv/parse.h` and `src/uenv/parse.cpp`
-3. write unit tests in `test/unit/parse.cpp`
+2. add a `parse` function in the relevant `parse.h`/`parse.cpp`
+3. write unit tests in the matching `test/unit/*.cpp`
 4. run the unit tests, and repeat the process until they pass.
 
+### Self-contained `src/oci`
+
+The `src/oci/` module is a native OCI registry client written to replace the
+external `oras` binary. It is intended to be reusable and independently testable,
+so it must depend **only** on `src/util/` and external libraries (fmt,
+nlohmann_json, libcurl, spdlog, zlib).
+
+`src/oci/` must **not** include or depend on `src/uenv/`, `src/site/`, or
+`src/cli/`. Code that both `uenv` and `oci` need (for example the lexer and the
+parsing scaffolding) lives in `src/util/` precisely so that `oci` never has to
+reach up into `uenv`:
+
+- `src/util/lex.*` — the shared tokenizer.
+- `src/util/parse.*` — `util::parse_error`, the `PARSE` macro, and
+  `util::parse_string`.
+
+Parsing in `src/oci` follows the same lexer-based idiom as the rest of the
+codebase; JSON documents are parsed with nlohmann_json, while
+`src/oci/parse.*` handles the character-level string types (digests, references,
+URLs, the bearer challenge).
+
+Enforcement: this command must return nothing.
+
+```bash
+grep -rn '#include <\(uenv\|site\|cli\)/' src/oci/
+```
 
 ### Environment Variables
 

meson.build

@@ -9,8 +9,6 @@ project('uenv', ['cpp'],
         meson_version: '>=1.4')
 version = meson.project_version()
 
-oras_version = get_option('oras_version')
-
 uenv_slurm_plugin = get_option('slurm_plugin')
 uenv_cli = get_option('cli')
 uenv_squashfs_mount = get_option('squashfs_mount')
@@ -38,6 +36,12 @@ toml_dep = declare_dependency(
   dependencies: toml_base,
   compile_args: ['-DTOML_EXCEPTIONS=0', '-DTOML_HEADER_ONLY=0'],
 )
+# zlib: needed for gzip handling in the native OCI registry client that
+# replaced the oras binary. Always built from the vendored wrapdb subproject
+# (never the system copy) and statically linked, so the binary is self-contained.
+# Built *before* curl so its override_dependency('zlib') is in place and curl
+# links the same vendored static zlib rather than the system copy.
+zlib_dep = subproject('zlib', default_options: ['warning_level=0', 'werror=false', 'default_library=static']).get_variable('zlib_dep')
 subproject('curl', default_options: ['werror=false', 'warning_level=0', 'tests=disabled', 'unittests=disabled', 'tool=disabled'])
 curl_dep = dependency('libcurl', required: true)
 barkeep_dep = declare_dependency(
@@ -56,7 +60,15 @@ lib_src = [
         'src/uenv/log.cpp',
         'src/uenv/meta.cpp',
         'src/uenv/mount.cpp',
-        'src/uenv/oras.cpp',
+        'src/oci/auth.cpp',
+        'src/oci/client.cpp',
+        'src/oci/digest.cpp',
+        'src/oci/manifest.cpp',
+        'src/oci/parse.cpp',
+        'src/oci/pull.cpp',
+        'src/oci/push.cpp',
+        'src/oci/reference.cpp',
+        'src/oci/util.cpp',
         'src/uenv/parse.cpp',
         'src/uenv/print.cpp',
         'src/uenv/repository.cpp',
@@ -69,7 +81,9 @@ lib_src = [
         'src/util/fs.cpp',
         'src/util/lex.cpp',
         'src/util/lustre.cpp',
+        'src/util/parse.cpp',
         'src/util/semver.cpp',
+        'src/util/sha256.cpp',
         'src/util/shell.cpp',
         'src/util/signal.cpp',
         'src/util/strings.cpp',
@@ -82,12 +96,12 @@ lib_uenv = static_library(
         'uenv',
         lib_src,
         include_directories: lib_inc,
-        dependencies: [curl_dep, sqlite3_dep, fmt_dep, spdlog_dep, json_dep, barkeep_dep, toml_dep],
+        dependencies: [curl_dep, zlib_dep, sqlite3_dep, fmt_dep, spdlog_dep, json_dep, barkeep_dep, toml_dep],
 )
 
 uenv_dep = declare_dependency(
         link_with: lib_uenv,
-        dependencies: [curl_dep, sqlite3_dep, fmt_dep, spdlog_dep, json_dep, barkeep_dep, libmount_dep, toml_dep],
+        dependencies: [curl_dep, zlib_dep, sqlite3_dep, fmt_dep, spdlog_dep, json_dep, barkeep_dep, libmount_dep, toml_dep],
         include_directories: lib_inc
 )
 
@@ -132,28 +146,6 @@ if uenv_cli
             ],
             install: true)
 
-    arch = 'amd64'
-    if host_machine.cpu_family() == 'aarch64'
-        arch = 'arm64'
-    endif
-    oras_url = 'https://github.com/oras-project/oras/releases/download/v@0@/oras_@0@_linux_@1@.tar.gz'.format(oras_version, arch)
-
-    oras_download = custom_target(
-        'oras-download',
-        output: 'oras.tar.gz',
-        command: ['wget', '--quiet', oras_url, '-O', '@OUTPUT@'],
-        install: false,
-    )
-
-    oras_extract = custom_target(
-        'oras-extract',
-        input: oras_download,
-        output: 'oras',
-        command: ['tar', 'xf', '@INPUT@', 'oras'],
-        install: true,
-        install_dir: get_option('libexecdir'),
-    )
-
     meson.add_install_script('./meson-scripts/install-bash-completion.sh')
 endif
 

meson_options.txt

@@ -4,6 +4,5 @@ option('slurm_plugin', type: 'boolean', value: true)
 option('cli', type: 'boolean', value: true)
 option('squashfs_mount', type: 'boolean', value: false)
 
-option('oras_version', type: 'string', value: '1.2.0')
 #option('slurm_version', type: 'string', value: '24.05.0')
 option('slurm_version', type: 'string', value: '0.00.0')

packaging/uenv.spec

@@ -35,5 +35,4 @@ echo "$REQ" > "$CNF"
 %{_bindir}/uenv
 %{_bindir}/squashfs-mount
 %attr(4755, root, root) %{_bindir}/squashfs-mount
-/usr/libexec/oras
 /usr/share/bash-completion/completions/uenv

src/cli/add_remove.cpp

@@ -1,5 +1,6 @@
 // vim: ts=4 sts=4 sw=4 et
 
+#include <memory>
 #include <string>
 
 #include <fmt/core.h>
@@ -103,7 +104,24 @@ int image_add(const image_add_args& args, const global_settings& settings) {
         sqfs = squashfs_image{env->sqfs_path, env->meta_path,
                               fmt::format("{}", env->record->sha)};
     } else {
-        sqfs = uenv::validate_squashfs_image(env->sqfs_path);
+        // hashing the image reads the whole file, so show progress for it. The
+        // bar is created on the first callback, once the size is known.
+        std::unique_ptr<uenv::transfer_bar> prepare_bar;
+        sqfs = uenv::validate_squashfs_image(
+            env->sqfs_path,
+            [&prepare_bar, &env](std::uint64_t done, std::uint64_t total) {
+                if (!prepare_bar) {
+                    prepare_bar = uenv::make_transfer_bar(
+                        total, fmt::format("preparing {}",
+                                           env->sqfs_path.filename().string()));
+                }
+                prepare_bar->update(done);
+            });
+        // stop the bar before anything else is printed, so that an error
+        // message does not land on the bar's line.
+        if (prepare_bar) {
+            prepare_bar->finish();
+        }
         if (!sqfs) {
             term::error("invalid source {}: {}", args.source, sqfs.error());
             return 1;

src/cli/build.cpp

@@ -100,7 +100,12 @@ int build(const build_args& args,
         return 1;
     }
 
-    auto recipe_tar_path = util::make_temp_dir() / "recipe.tar.gz";
+    auto tmp_dir = util::make_temp_dir();
+    if (!tmp_dir) {
+        term::error("{}", tmp_dir.error());
+        return 1;
+    }
+    auto recipe_tar_path = *tmp_dir / "recipe.tar.gz";
     auto proc = util::run({"env", "--chdir", recipe_path.string(), "tar",
                            "--dereference", "-czf", recipe_tar_path, "."});
     if (proc->rvalue() > 0) {