From 5c4f8ced51ab734c87e53632ae3ed3ff5498631f Mon Sep 17 00:00:00 2001 From: bcumming Date: Mon, 29 Jun 2026 15:22:05 +0200 Subject: [PATCH 01/29] vendor zlib, add sha256 implementation --- .gitignore | 2 + meson.build | 11 +- src/cli/util.cpp | 15 +-- src/util/sha256.cpp | 243 ++++++++++++++++++++++++++++++++++++++++++ src/util/sha256.h | 58 ++++++++++ subprojects/zlib.wrap | 14 +++ test/meson.build | 1 + test/unit/sha256.cpp | 95 +++++++++++++++++ 8 files changed, 427 insertions(+), 12 deletions(-) create mode 100644 src/util/sha256.cpp create mode 100644 src/util/sha256.h create mode 100644 subprojects/zlib.wrap create mode 100644 test/unit/sha256.cpp diff --git a/.gitignore b/.gitignore index 6e96bf53..fbb331f6 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,8 @@ subprojects/spdlog-* subprojects/sqlite-amalgamation-* subprojects/nlohmann_json-* subprojects/tomlplusplus-* +subprojects/zlib-* +subprojects/wrapdb.json subprojects/.wraplock # logs and artifacts generated by packaging workflows diff --git a/meson.build b/meson.build index 879a3a6f..54e713ed 100644 --- a/meson.build +++ b/meson.build @@ -38,6 +38,12 @@ toml_dep = declare_dependency( dependencies: toml_base, compile_args: ['-DTOML_EXCEPTIONS=0', '-DTOML_HEADER_ONLY=0'], ) +# zlib: needed for gzip handling in the native OCI registry client that will +# replace the oras binary. Always built from the vendored wrapdb subproject +# (never the system copy) and statically linked, so the binary is self-contained. +# Built *before* curl so its override_dependency('zlib') is in place and curl +# links the same vendored static zlib rather than the system copy. +zlib_dep = subproject('zlib', default_options: ['warning_level=0', 'werror=false', 'default_library=static']).get_variable('zlib_dep') subproject('curl', default_options: ['werror=false', 'warning_level=0', 'tests=disabled', 'unittests=disabled', 'tool=disabled']) curl_dep = dependency('libcurl', required: true) barkeep_dep = declare_dependency( @@ -70,6 +76,7 @@ lib_src = [ 'src/util/lex.cpp', 'src/util/lustre.cpp', 'src/util/semver.cpp', + 'src/util/sha256.cpp', 'src/util/shell.cpp', 'src/util/signal.cpp', 'src/util/strings.cpp', @@ -82,12 +89,12 @@ lib_uenv = static_library( 'uenv', lib_src, include_directories: lib_inc, - dependencies: [curl_dep, sqlite3_dep, fmt_dep, spdlog_dep, json_dep, barkeep_dep, toml_dep], + dependencies: [curl_dep, zlib_dep, sqlite3_dep, fmt_dep, spdlog_dep, json_dep, barkeep_dep, toml_dep], ) uenv_dep = declare_dependency( link_with: lib_uenv, - dependencies: [curl_dep, sqlite3_dep, fmt_dep, spdlog_dep, json_dep, barkeep_dep, libmount_dep, toml_dep], + dependencies: [curl_dep, zlib_dep, sqlite3_dep, fmt_dep, spdlog_dep, json_dep, barkeep_dep, libmount_dep, toml_dep], include_directories: lib_inc ) diff --git a/src/cli/util.cpp b/src/cli/util.cpp index d4230334..c3938c15 100644 --- a/src/cli/util.cpp +++ b/src/cli/util.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -82,19 +83,13 @@ validate_squashfs_image(const std::string& path) { spdlog::info("no meta data in {}", img.sqfs); } - auto proc = util::run({"sha256sum", img.sqfs.string()}); - if (!proc) { - spdlog::error("{}", proc.error()); + auto hash = util::sha256_file(img.sqfs); + if (!hash) { + spdlog::error("{}", hash.error()); return util::unexpected{fmt::format( "unable to calculate sha256 of squashfs file {}", img.sqfs)}; } - auto success = proc->wait(); - if (success != 0) { - return util::unexpected{fmt::format( - "unable to calculate sha256 of squashfs file {}", img.sqfs)}; - } - auto raw = *proc->out.getline(); - img.hash = raw.substr(0, 64); + img.hash = hash->hex(); return img; } diff --git a/src/util/sha256.cpp b/src/util/sha256.cpp new file mode 100644 index 00000000..0d71dc64 --- /dev/null +++ b/src/util/sha256.cpp @@ -0,0 +1,243 @@ +#include +#include +#include +#include +#include + +#include + +#include "sha256.h" + +namespace util { + +namespace { + +// SHA-256 round constants (first 32 bits of the fractional parts of the cube +// roots of the first 64 primes), FIPS 180-4 §4.2.2. +constexpr uint32_t K[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, + 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, + 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, + 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, + 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, + 0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2}; + +inline uint32_t rotr(uint32_t x, uint32_t n) { + return (x >> n) | (x << (32 - n)); +} + +// The real digest state. Lives only in this translation unit; callers see it as +// the opaque sha256_state blob. +struct sha256_impl { + uint32_t state[8]; + uint64_t bitlen; + uint8_t buffer[64]; + std::size_t buffer_len; + + void init(); + void update(const uint8_t* p, std::size_t len); + sha256_digest final(); + void process_block(const uint8_t block[64]); +}; + +static_assert(sizeof(sha256_impl) <= sizeof(sha256_state::opaque), + "sha256_state::opaque too small for the implementation state"); +static_assert(alignof(sha256_impl) <= alignof(sha256_state), + "sha256_state under-aligned for the implementation state"); + +// view the opaque blob as the implementation state (lifetime begun in +// sha256_init via placement new). +inline sha256_impl& impl(sha256_state& s) { + return *std::launder(reinterpret_cast(s.opaque)); +} + +void sha256_impl::init() { + // initial hash values: fractional parts of the square roots of the first 8 + // primes, FIPS 180-4 §5.3.3. + state[0] = 0x6a09e667; + state[1] = 0xbb67ae85; + state[2] = 0x3c6ef372; + state[3] = 0xa54ff53a; + state[4] = 0x510e527f; + state[5] = 0x9b05688c; + state[6] = 0x1f83d9ab; + state[7] = 0x5be0cd19; + bitlen = 0; + buffer_len = 0; +} + +void sha256_impl::process_block(const uint8_t block[64]) { + uint32_t w[64]; + for (int i = 0; i < 16; ++i) { + w[i] = (static_cast(block[i * 4]) << 24) | + (static_cast(block[i * 4 + 1]) << 16) | + (static_cast(block[i * 4 + 2]) << 8) | + (static_cast(block[i * 4 + 3])); + } + for (int i = 16; i < 64; ++i) { + uint32_t s0 = + rotr(w[i - 15], 7) ^ rotr(w[i - 15], 18) ^ (w[i - 15] >> 3); + uint32_t s1 = rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ (w[i - 2] >> 10); + w[i] = w[i - 16] + s0 + w[i - 7] + s1; + } + + uint32_t a = state[0], b = state[1], c = state[2], d = state[3]; + uint32_t e = state[4], f = state[5], g = state[6], h = state[7]; + + for (int i = 0; i < 64; ++i) { + uint32_t S1 = rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25); + uint32_t ch = (e & f) ^ (~e & g); + uint32_t temp1 = h + S1 + ch + K[i] + w[i]; + uint32_t S0 = rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22); + uint32_t maj = (a & b) ^ (a & c) ^ (b & c); + uint32_t temp2 = S0 + maj; + + h = g; + g = f; + f = e; + e = d + temp1; + d = c; + c = b; + b = a; + a = temp1 + temp2; + } + + state[0] += a; + state[1] += b; + state[2] += c; + state[3] += d; + state[4] += e; + state[5] += f; + state[6] += g; + state[7] += h; +} + +void sha256_impl::update(const uint8_t* p, std::size_t len) { + bitlen += static_cast(len) * 8; + + // top up a partially-filled buffer first + if (buffer_len > 0) { + std::size_t need = 64 - buffer_len; + std::size_t take = len < need ? len : need; + std::memcpy(buffer + buffer_len, p, take); + buffer_len += take; + p += take; + len -= take; + if (buffer_len == 64) { + process_block(buffer); + buffer_len = 0; + } + } + + // process whole blocks straight from the input + while (len >= 64) { + process_block(p); + p += 64; + len -= 64; + } + + // stash the remainder + if (len > 0) { + std::memcpy(buffer + buffer_len, p, len); + buffer_len += len; + } +} + +sha256_digest sha256_impl::final() { + uint64_t final_bitlen = bitlen; + + // append the 0x80 padding byte, then zeros up to a 56-byte boundary, then + // the 64-bit big-endian length. + uint8_t pad = 0x80; + update(&pad, 1); + + uint8_t zero = 0x00; + while (buffer_len != 56) { + update(&zero, 1); + } + + uint8_t lenbytes[8]; + for (int i = 0; i < 8; ++i) { + lenbytes[i] = static_cast(final_bitlen >> (56 - i * 8)); + } + update(lenbytes, 8); // fills the block; update() flushes it + + sha256_digest digest{}; + for (int i = 0; i < 8; ++i) { + digest.bytes[i * 4] = static_cast(state[i] >> 24); + digest.bytes[i * 4 + 1] = static_cast(state[i] >> 16); + digest.bytes[i * 4 + 2] = static_cast(state[i] >> 8); + digest.bytes[i * 4 + 3] = static_cast(state[i]); + } + return digest; +} + +} // namespace + +std::string sha256_digest::hex() const { + std::string out; + out.reserve(64); + for (std::uint8_t b : bytes) { + fmt::format_to(std::back_inserter(out), "{:02x}", b); + } + return out; +} + +void sha256_init(sha256_state& state) { + auto* p = new (state.opaque) sha256_impl; + p->init(); +} + +void sha256_update(sha256_state& state, std::span data) { + impl(state).update(reinterpret_cast(data.data()), + data.size()); +} + +sha256_digest sha256_final(sha256_state& state) { + return impl(state).final(); +} + +sha256_digest sha256_bytes(std::span data) { + sha256_state s; + sha256_init(s); + sha256_update(s, data); + return sha256_final(s); +} + +sha256_digest sha256_string(std::string_view text) { + return sha256_bytes( + {reinterpret_cast(text.data()), text.size()}); +} + +util::expected +sha256_file(const std::filesystem::path& path) { + std::FILE* f = std::fopen(path.c_str(), "rb"); + if (f == nullptr) { + return util::unexpected{fmt::format("unable to open {} for reading: {}", + path.string(), + std::strerror(errno))}; + } + + sha256_state s; + sha256_init(s); + std::array buf; + std::size_t n; + while ((n = std::fread(buf.data(), 1, buf.size(), f)) > 0) { + sha256_update(s, std::span{buf.data(), n}); + } + + if (std::ferror(f)) { + std::fclose(f); + return util::unexpected{fmt::format("error reading {}", path.string())}; + } + std::fclose(f); + + return sha256_final(s); +} + +} // namespace util diff --git a/src/util/sha256.h b/src/util/sha256.h new file mode 100644 index 00000000..4adbb992 --- /dev/null +++ b/src/util/sha256.h @@ -0,0 +1,58 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace util { + +// A SHA-256 digest: 32 raw bytes, plus a canonical lowercase-hex rendering. +struct sha256_digest { + std::array bytes; + + // 64-character lowercase hex string. + std::string hex() const; + + friend bool operator==(const sha256_digest&, + const sha256_digest&) = default; +}; + +// --- low-level streaming interface -------------------------------------- +// +// An opaque state blob you sha256_init() once, feed bytes into with +// sha256_update(), then sha256_final(). Stack-allocatable and trivially +// copyable; the internal layout lives entirely in sha256.cpp, so the FIPS +// implementation never leaks into this header. Use this when data must be +// digested incrementally (e.g. streaming a multi-GB squashfs) without holding +// it all in memory. +struct sha256_state { + // opaque storage - do not inspect or modify directly. sized and aligned to + // hold the implementation state (asserted in sha256.cpp). + alignas(std::uint64_t) std::byte opaque[112]; +}; + +void sha256_init(sha256_state& state); +void sha256_update(sha256_state& state, std::span data); +sha256_digest sha256_final(sha256_state& state); + +// --- convenience one-shots ---------------------------------------------- + +// digest an in-memory byte buffer. +sha256_digest sha256_bytes(std::span data); + +// digest text - a thin wrapper for the common string case. +sha256_digest sha256_string(std::string_view text); + +// digest a file by streaming it in fixed-size chunks, so large files never +// need to be held in memory. returns an error message if the file is +// unreadable. +util::expected +sha256_file(const std::filesystem::path& path); + +} // namespace util diff --git a/subprojects/zlib.wrap b/subprojects/zlib.wrap new file mode 100644 index 00000000..0626401a --- /dev/null +++ b/subprojects/zlib.wrap @@ -0,0 +1,14 @@ +[wrap-file] +directory = zlib-1.3.2 +source_url = https://zlib.net/zlib-1.3.2.tar.xz +source_fallback_url = https://wrapdb.mesonbuild.com/v2/zlib_1.3.2-1/get_source/zlib-1.3.2.tar.xz +source_filename = zlib-1.3.2.tar.xz +source_hash = d7a0654783a4da529d1bb793b7ad9c3318020af77667bcae35f95d0e42a792f3 +patch_filename = zlib_1.3.2-1_patch.zip +patch_url = https://wrapdb.mesonbuild.com/v2/zlib_1.3.2-1/get_patch +patch_fallback_url = https://github.com/mesonbuild/wrapdb/releases/download/zlib_1.3.2-1/zlib_1.3.2-1_patch.zip +patch_hash = 5ae7a2e92f823df118cfb8c1b23d94e3117864392b3446581d669049b2fba6dd +wrapdb_version = 1.3.2-1 + +[provide] +dependency_names = zlib diff --git a/test/meson.build b/test/meson.build index 6b2a0c4e..39037d6d 100644 --- a/test/meson.build +++ b/test/meson.build @@ -74,6 +74,7 @@ unit_src = [ 'unit/strings.cpp', 'unit/repository.cpp', 'unit/settings.cpp', + 'unit/sha256.cpp', 'unit/subprocess.cpp', 'unit/telemetry.cpp', ] diff --git a/test/unit/sha256.cpp b/test/unit/sha256.cpp new file mode 100644 index 00000000..9281a325 --- /dev/null +++ b/test/unit/sha256.cpp @@ -0,0 +1,95 @@ +#include +#include +#include +#include + +#include + +#include + +namespace { +// digest a text chunk through the low-level streaming interface. +std::span as_bytes(std::string_view s) { + return {reinterpret_cast(s.data()), s.size()}; +} +} // namespace + +TEST_CASE("sha256 known vectors", "[sha256]") { + // canonical FIPS 180-4 / NIST test vectors + REQUIRE(util::sha256_string("").hex() == + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); + REQUIRE(util::sha256_string("abc").hex() == + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + REQUIRE( + util::sha256_string( + "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq") + .hex() == + "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1"); +} + +TEST_CASE("sha256 zero-byte input", "[sha256]") { + const std::string empty = + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + + // empty (zero-length) input must hash to the canonical empty digest, via + // every interface that accepts data. + REQUIRE(util::sha256_string("").hex() == empty); + REQUIRE(util::sha256_bytes({}).hex() == empty); + + // low-level: init then final with no update at all. + util::sha256_state s; + util::sha256_init(s); + REQUIRE(util::sha256_final(s).hex() == empty); + + // low-level: an explicit zero-length update is a no-op. + util::sha256_state s2; + util::sha256_init(s2); + util::sha256_update(s2, {}); + REQUIRE(util::sha256_final(s2).hex() == empty); + + // a single NUL byte (0x00) is distinct from empty input and has its own + // known digest. + const std::byte nul{0x00}; + REQUIRE(util::sha256_bytes({&nul, 1}).hex() == + "6e340b9cffb37a989ca544e6bb780a2c78901d3fb33738768511a30617afa01d"); +} + +TEST_CASE("sha256_digest carries raw bytes", "[sha256]") { + auto d = util::sha256_string("abc"); + REQUIRE(d.bytes[0] == 0xba); + REQUIRE(d.bytes[1] == 0x78); + REQUIRE(d.bytes[31] == 0xad); + REQUIRE(d == util::sha256_string("abc")); + REQUIRE_FALSE(d == util::sha256_string("abd")); +} + +TEST_CASE("sha256 multi-block and incremental", "[sha256]") { + // a million 'a' characters -> known NIST vector. fed in chunks to exercise + // the streaming/buffering path of the low-level interface. + const std::string expected = + "cdc76e5c9914fb9281a1c7e284d73e67f1809a48a497200e046d39ccc7112cd0"; + + util::sha256_state s; + util::sha256_init(s); + std::string chunk(1000, 'a'); + for (int i = 0; i < 1000; ++i) { + util::sha256_update(s, as_bytes(chunk)); + } + REQUIRE(util::sha256_final(s).hex() == expected); + + // same input in a single buffer must match + REQUIRE(util::sha256_string(std::string(1000000, 'a')).hex() == expected); +} + +TEST_CASE("sha256 update splits do not change the digest", "[sha256]") { + const std::string msg = "the quick brown fox jumps over the lazy dog"; + const std::string whole = util::sha256_string(msg).hex(); + + for (std::size_t split = 0; split <= msg.size(); ++split) { + util::sha256_state s; + util::sha256_init(s); + util::sha256_update(s, as_bytes(std::string_view{msg}.substr(0, split))); + util::sha256_update(s, as_bytes(std::string_view{msg}.substr(split))); + REQUIRE(util::sha256_final(s).hex() == whole); + } +} From 35bd64fe2eb967b84a05263df10e07413cde112d Mon Sep 17 00:00:00 2001 From: bcumming Date: Mon, 29 Jun 2026 15:46:07 +0200 Subject: [PATCH 02/29] generalise curl code and add features for oras replacement. --- src/util/curl.cpp | 373 +++++++++++++++++++++++++++++---------------- src/util/curl.h | 65 ++++++++ test/meson.build | 1 + test/unit/curl.cpp | 52 +++++++ 4 files changed, 360 insertions(+), 131 deletions(-) create mode 100644 test/unit/curl.cpp diff --git a/src/util/curl.cpp b/src/util/curl.cpp index d2666308..f76685f1 100644 --- a/src/util/curl.cpp +++ b/src/util/curl.cpp @@ -1,4 +1,6 @@ +#include #include +#include #include #include @@ -15,6 +17,59 @@ namespace util { namespace curl { +namespace { + +std::string to_lower(std::string_view s) { + std::string out; + out.reserve(s.size()); + for (char c : s) { + out.push_back(static_cast( + std::tolower(static_cast(c)))); + } + return out; +} + +std::string_view trim(std::string_view s) { + auto is_space = [](char c) { + return std::isspace(static_cast(c)) != 0; + }; + while (!s.empty() && is_space(s.front())) { + s.remove_prefix(1); + } + while (!s.empty() && is_space(s.back())) { + s.remove_suffix(1); + } + return s; +} + +} // namespace + +std::optional headers::get(std::string_view name) const { + if (auto it = entries.find(to_lower(name)); it != entries.end()) { + return it->second; + } + return std::nullopt; +} + +void headers::set(std::string_view name, std::string value) { + entries[to_lower(name)] = std::move(value); +} + +void parse_header_line(headers& h, std::string_view line) { + line = trim(line); // drops trailing CRLF and surrounding whitespace + auto colon = line.find(':'); + // no colon -> status line ("HTTP/1.1 200 OK") or blank separator: ignore. + if (colon == std::string_view::npos) { + return; + } + auto name = trim(line.substr(0, colon)); + auto value = trim(line.substr(colon + 1)); + if (name.empty()) { + return; + } + h.set(name, std::string{value}); +} + std::string http_message(long code) { const char* default_message = "internal error contacting a network service - please create a CSCS " @@ -50,101 +105,47 @@ size_t memory_callback(void* source, size_t size, size_t n, void* target) { expected post(const std::string& data, std::string url, std::optional content_type, long timeout_ms) { - char errbuf[CURL_ERROR_SIZE]; - errbuf[0] = 0; spdlog::trace("curl::post enter"); - CURL* h = curl_easy_init(); - if (!h) { - return unexpected{ - error{CURLE_FAILED_INIT, "unable to initialise curl"}}; - } - auto _ = defer([h]() { curl_easy_cleanup(h); }); - spdlog::trace("curl::upload easy init"); - - // configure error message buffer - CURL_EASY(curl_easy_setopt(h, CURLOPT_ERRORBUFFER, errbuf)); - // Set up curl options... - CURL_EASY(curl_easy_setopt(h, CURLOPT_URL, url.c_str())); - spdlog::trace("curl::post set url {}", url); - // ... other setup ... - CURL_EASY(curl_easy_setopt(h, CURLOPT_POSTFIELDS, data.c_str())); - CURL_EASY(curl_easy_setopt(h, CURLOPT_POSTFIELDSIZE, data.size())); - spdlog::trace("curl::post set data {}", data); - // Headers + request req; + req.url = std::move(url); + req.method = http_method::post; + req.body = data; if (content_type) { - struct curl_slist* headers = NULL; - headers = curl_slist_append( - headers, fmt::format("Content-Type: {}", *content_type).c_str()); - CURL_EASY(curl_easy_setopt(h, CURLOPT_HTTPHEADER, headers)); + req.header_lines.push_back( + fmt::format("Content-Type: {}", *content_type)); + } + // preserve legacy behaviour: caller-supplied total timeout, libcurl's + // default connect timeout, and no redirect following. + req.timeout_ms = timeout_ms; + req.connect_timeout_ms = 300000L; // libcurl default + req.follow_redirects = false; + + auto resp = perform(req); + if (!resp) { + return unexpected{resp.error()}; } - - // Set callback function to capture response - CURL_EASY(curl_easy_setopt(h, CURLOPT_WRITEFUNCTION, memory_callback)); - // we pass our 'chunk' struct to the callback function - std::vector result; - result.reserve(200000); - CURL_EASY(curl_easy_setopt(h, CURLOPT_WRITEDATA, (void*)&result)); - spdlog::trace("curl::post set memory target"); - - // set timeouts - CURL_EASY(curl_easy_setopt(h, CURLOPT_TIMEOUT_MS, timeout_ms)); - spdlog::trace("curl::get set timeout"); - - // Perform the request - CURL_EASY(curl_easy_perform(h)); - spdlog::trace("curl::post finished and retrieved data of size {}", - result.size()); - - return std::string{result.data(), result.data() + result.size()}; + resp->body.size()); + return resp->body; } expected get(std::string url) { - char errbuf[CURL_ERROR_SIZE]; - errbuf[0] = 0; - - auto h = curl_easy_init(); - if (!h) { - return unexpected{ - error{CURLE_FAILED_INIT, "unable to initialise curl"}}; + request req; + req.url = std::move(url); + req.method = http_method::get; + // wait up to 4s to connect and no more than 5s for the whole operation. + req.connect_timeout_ms = 4000L; + req.timeout_ms = 5000L; + req.follow_redirects = false; + + auto resp = perform(req); + if (!resp) { + return unexpected{resp.error()}; } - auto _ = defer([h]() { curl_easy_cleanup(h); }); - - CURL_EASY(curl_easy_setopt(h, CURLOPT_ERRORBUFFER, errbuf)); - - CURL_EASY(curl_easy_setopt(h, CURLOPT_URL, url.c_str())); - spdlog::trace("curl::get set url {}", url); - - // send all data to this function - CURL_EASY(curl_easy_setopt(h, CURLOPT_WRITEFUNCTION, memory_callback)); - spdlog::trace("curl::get set memory callback"); - - // we pass our 'chunk' struct to the callback function - std::vector result; - // when this was written, calls to the jfrog API returned roughly 100k - // bytes - result.reserve(200000); - CURL_EASY(curl_easy_setopt(h, CURLOPT_WRITEDATA, (void*)&result)); - spdlog::trace("curl::get set memory target"); - - // some servers do not like requests that are made without a user-agent - // field, so we provide one - CURL_EASY(curl_easy_setopt(h, CURLOPT_USERAGENT, "libcurl-agent/1.0")); - spdlog::trace("curl::get set user agent"); - - // set a reasonable time out - wait 1 second for connection, and no more - // than 2 seconds for the whole operation. - CURL_EASY(curl_easy_setopt(h, CURLOPT_CONNECTTIMEOUT_MS, 4000L)); - CURL_EASY(curl_easy_setopt(h, CURLOPT_TIMEOUT_MS, 5000L)); - spdlog::trace("curl::get set timeout"); - - CURL_EASY(curl_easy_perform(h)); - spdlog::trace("curl::get finished and retrieved data of size {}", - result.size()); - - return std::string{result.data(), result.data() + result.size()}; + resp->body.size()); + return resp->body; } expected upload(std::string url, @@ -233,70 +234,180 @@ expected upload(std::string url, expected del(std::string url, std::string username, std::string token) { - char errbuf[CURL_ERROR_SIZE]; - errbuf[0] = 0; + request req; + req.url = url; + req.method = http_method::del; + req.username = username; + req.password = token; + req.connect_timeout_ms = 1000L; + req.timeout_ms = 10000L; + req.follow_redirects = false; + + auto resp = perform(req); + if (!resp) { + return unexpected{resp.error()}; + } + spdlog::trace("curl::del http_code: {}", resp->status); - auto h = curl_easy_init(); - if (!h) { - return unexpected{ - error{CURLE_FAILED_INIT, "unable to initialise curl"}}; + if (resp->status >= 400) { + return unexpected{error{ + CURLE_HTTP_RETURNED_ERROR, + fmt::format("{}: {}", resp->status, http_message(resp->status))}}; } - auto _ = defer([h]() { curl_easy_cleanup(h); }); - CURL_EASY(curl_easy_setopt(h, CURLOPT_ERRORBUFFER, errbuf)); + spdlog::info("curl -X DELETE -u {}:{} {}", username, + std::string(token.size(), 'X'), url); - CURL_EASY(curl_easy_setopt(h, CURLOPT_URL, url.c_str())); - spdlog::trace("curl::get set url {}", url); + spdlog::trace("curl::del successfully deleted {}", url); - // send all data to this function - CURL_EASY(curl_easy_setopt(h, CURLOPT_WRITEFUNCTION, memory_callback)); - spdlog::trace("curl::get set memory callback"); + return {}; +} - // we pass our 'chunk' struct to the callback function - std::vector result; - result.reserve(10000); - CURL_EASY(curl_easy_setopt(h, CURLOPT_WRITEDATA, (void*)&result)); - spdlog::trace("curl::get set memory target"); +namespace { - // some servers do not like requests that are made without a user-agent - // field, so we provide one - CURL_EASY(curl_easy_setopt(h, CURLOPT_USERAGENT, "libcurl-agent/1.0")); - spdlog::trace("curl::get set user agent"); +size_t header_callback(char* buffer, size_t size, size_t nitems, + void* userdata) { + const size_t total = size * nitems; + auto& h = *static_cast(userdata); + parse_header_line(h, std::string_view{buffer, total}); + return total; +} - // Specify the HTTP method as DELETE - CURL_EASY(curl_easy_setopt(h, CURLOPT_CUSTOMREQUEST, "DELETE")); +const char* method_name(http_method m) { + switch (m) { + case http_method::get: + return "GET"; + case http_method::head: + return "HEAD"; + case http_method::post: + return "POST"; + case http_method::put: + return "PUT"; + case http_method::patch: + return "PATCH"; + case http_method::del: + return "DELETE"; + } + return "GET"; +} - // Set the username and password for basic authentication - CURL_EASY(curl_easy_setopt(h, CURLOPT_USERNAME, username.c_str())); - CURL_EASY(curl_easy_setopt(h, CURLOPT_PASSWORD, token.c_str())); - spdlog::trace("curl::get set credentials"); +} // namespace - CURL_EASY(curl_easy_setopt(h, CURLOPT_CONNECTTIMEOUT_MS, 1000L)); - CURL_EASY(curl_easy_setopt(h, CURLOPT_TIMEOUT_MS, 10000L)); - spdlog::trace("curl::get set timeout"); +expected perform(const request& req) { + char errbuf[CURL_ERROR_SIZE]; + errbuf[0] = 0; + spdlog::trace("curl::perform {} {}", method_name(req.method), req.url); - CURL_EASY(curl_easy_perform(h)); + CURL* h = curl_easy_init(); + if (!h) { + return unexpected{error{CURLE_FAILED_INIT, "unable to initialise curl"}}; + } + auto cleanup_handle = defer([h]() { curl_easy_cleanup(h); }); - // Get the HTTP response code - long http_code = 0; - CURL_EASY(curl_easy_getinfo(h, CURLINFO_RESPONSE_CODE, &http_code)); - spdlog::trace("curl::upload http_code: {}", http_code); + CURL_EASY(curl_easy_setopt(h, CURLOPT_ERRORBUFFER, errbuf)); + CURL_EASY(curl_easy_setopt(h, CURLOPT_URL, req.url.c_str())); + CURL_EASY(curl_easy_setopt(h, CURLOPT_USERAGENT, "libcurl-agent/1.0")); - // store stdout - std::string curl_stdout{result.data(), result.data() + result.size()}; + // method + switch (req.method) { + case http_method::get: + CURL_EASY(curl_easy_setopt(h, CURLOPT_HTTPGET, 1L)); + break; + case http_method::head: + CURL_EASY(curl_easy_setopt(h, CURLOPT_NOBODY, 1L)); + break; + default: + CURL_EASY( + curl_easy_setopt(h, CURLOPT_CUSTOMREQUEST, method_name(req.method))); + break; + } - if (http_code >= 400) { - return unexpected{ - error{CURLE_HTTP_RETURNED_ERROR, - fmt::format("{}: {}", http_code, http_message(http_code))}}; + // redirect following, opt-in only. when enabled, guard it: cap the redirect + // count to avoid loops, and restrict redirects to https so a redirect can't + // downgrade the transport. credentials are NOT carried across hosts - + // CURLOPT_UNRESTRICTED_AUTH is deliberately left unset (libcurl default). + if (req.follow_redirects) { + CURL_EASY(curl_easy_setopt(h, CURLOPT_FOLLOWLOCATION, 1L)); + CURL_EASY(curl_easy_setopt(h, CURLOPT_MAXREDIRS, 10L)); + CURL_EASY( + curl_easy_setopt(h, CURLOPT_REDIR_PROTOCOLS_STR, "https")); } - spdlog::info("curl -X DELETE -u {}:{} {}", username, - std::string(token.size(), 'X'), url); + // assemble request headers + struct curl_slist* slist = nullptr; + auto cleanup_slist = defer([&slist]() { + if (slist) { + curl_slist_free_all(slist); + } + }); + for (const auto& line : req.header_lines) { + slist = curl_slist_append(slist, line.c_str()); + } + if (req.bearer_token) { + slist = curl_slist_append( + slist, fmt::format("Authorization: Bearer {}", *req.bearer_token) + .c_str()); + } + if (slist) { + CURL_EASY(curl_easy_setopt(h, CURLOPT_HTTPHEADER, slist)); + } - spdlog::trace("curl::get successfully deleted {}", url); + // basic auth via libcurl (credentials never reach a command line / argv) + if (req.username) { + CURL_EASY(curl_easy_setopt(h, CURLOPT_USERNAME, req.username->c_str())); + } + if (req.password) { + CURL_EASY(curl_easy_setopt(h, CURLOPT_PASSWORD, req.password->c_str())); + } - return {}; + // request body. file upload takes precedence over an in-memory body. + FILE* upload = nullptr; + auto cleanup_upload = defer([&upload]() { + if (upload) { + fclose(upload); + } + }); + if (req.upload_file) { + upload = fopen(req.upload_file->c_str(), "rb"); + if (!upload) { + return unexpected{ + error{CURLE_READ_ERROR, + fmt::format("unable to open {} for upload", + req.upload_file->string())}}; + } + curl_off_t file_size = std::filesystem::file_size(*req.upload_file); + CURL_EASY(curl_easy_setopt(h, CURLOPT_UPLOAD, 1L)); + CURL_EASY(curl_easy_setopt(h, CURLOPT_READDATA, upload)); + CURL_EASY(curl_easy_setopt(h, CURLOPT_INFILESIZE_LARGE, file_size)); + } else if (req.body) { + CURL_EASY(curl_easy_setopt(h, CURLOPT_POSTFIELDS, req.body->data())); + CURL_EASY(curl_easy_setopt(h, CURLOPT_POSTFIELDSIZE, + (long)req.body->size())); + } + + // capture body + std::vector body; + body.reserve(200000); + CURL_EASY(curl_easy_setopt(h, CURLOPT_WRITEFUNCTION, memory_callback)); + CURL_EASY(curl_easy_setopt(h, CURLOPT_WRITEDATA, (void*)&body)); + + // capture response headers + response resp; + CURL_EASY(curl_easy_setopt(h, CURLOPT_HEADERFUNCTION, header_callback)); + CURL_EASY(curl_easy_setopt(h, CURLOPT_HEADERDATA, (void*)&resp.headers)); + + CURL_EASY(curl_easy_setopt(h, CURLOPT_CONNECTTIMEOUT_MS, + req.connect_timeout_ms)); + CURL_EASY(curl_easy_setopt(h, CURLOPT_TIMEOUT_MS, req.timeout_ms)); + + CURL_EASY(curl_easy_perform(h)); + + CURL_EASY(curl_easy_getinfo(h, CURLINFO_RESPONSE_CODE, &resp.status)); + resp.body.assign(body.data(), body.size()); + spdlog::trace("curl::perform got status {} ({} body bytes)", resp.status, + resp.body.size()); + + return resp; } } // namespace curl diff --git a/src/util/curl.h b/src/util/curl.h index 5f17a8a3..7be463c0 100644 --- a/src/util/curl.h +++ b/src/util/curl.h @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include @@ -17,6 +19,69 @@ struct error { std::string message; }; +// --- low-level request/response primitive ------------------------------- +// +// A single configurable HTTP request used to build the native OCI registry +// client (the token dance, blob HEAD/PUT, manifest PUT/GET, referrers/tags). +// Unlike the convenience helpers below, perform() does NOT treat HTTP >= 400 as +// an error: a 401 auth challenge or a 404 blob-miss are normal steps in the OCI +// flow, so the status and response headers are always handed back to the caller. + +enum class http_method { get, head, post, put, patch, del }; + +// Case-insensitive view of HTTP response headers. Header names are stored +// lower-cased (HTTP field names are case-insensitive); the last value seen for a +// given name wins. +struct headers { + std::unordered_map entries; + + std::optional get(std::string_view name) const; + void set(std::string_view name, std::string value); +}; + +struct request { + std::string url; + http_method method = http_method::get; + + // extra request headers, each a raw "Name: value" line. + std::vector header_lines; + // when set, an "Authorization: Bearer " header is added. + std::optional bearer_token; + // HTTP basic-auth credentials (sent via libcurl, never on a command line). + std::optional username; + std::optional password; + + // request body. at most one of these should be set (used by put/post/patch): + // - body: an in-memory payload (e.g. a manifest blob) + // - upload_file: stream a file as the body (e.g. a squashfs layer) + std::optional body; + std::optional upload_file; + + // follow 3xx redirects. off by default (matching libcurl), since following + // is an SSRF/credential-leak/protocol-downgrade surface that only specific + // operations need (e.g. a blob GET that a registry 307-redirects to cloud + // storage). when enabled, perform() caps the redirect count and restricts + // redirects to https. note the blob *upload* Location arrives on a 202 (not + // a 3xx), so it is captured via the response headers regardless. + bool follow_redirects = false; + + long connect_timeout_ms = 5000; + // total operation timeout; 0 means no limit (needed for large uploads). + long timeout_ms = 0; +}; + +struct response { + long status = 0; + std::string body; + curl::headers headers; +}; + +expected perform(const request& req); + +// parse a single raw HTTP header line ("Name: value\r\n") into a headers map. +// status lines and blank separators are ignored. exposed for unit testing. +void parse_header_line(headers& h, std::string_view line); + std::string curl_get(std::string url); expected diff --git a/test/meson.build b/test/meson.build index 39037d6d..a993a2df 100644 --- a/test/meson.build +++ b/test/meson.build @@ -61,6 +61,7 @@ setup_suite = custom_target ( ) unit_src = [ + 'unit/curl.cpp', 'unit/dates.cpp', 'unit/env.cpp', 'unit/envvars.cpp', diff --git a/test/unit/curl.cpp b/test/unit/curl.cpp new file mode 100644 index 00000000..899ce265 --- /dev/null +++ b/test/unit/curl.cpp @@ -0,0 +1,52 @@ +#include + +#include + +TEST_CASE("curl headers are case-insensitive", "[curl]") { + util::curl::headers h; + h.set("Content-Type", "application/json"); + + REQUIRE(h.get("Content-Type") == "application/json"); + REQUIRE(h.get("content-type") == "application/json"); + REQUIRE(h.get("CONTENT-TYPE") == "application/json"); + REQUIRE(h.get("missing") == std::nullopt); + + // last value seen for a name wins + h.set("content-type", "text/plain"); + REQUIRE(h.get("Content-Type") == "text/plain"); +} + +TEST_CASE("curl parse_header_line", "[curl]") { + util::curl::headers h; + + // the OCI-relevant headers the registry client needs to capture + util::curl::parse_header_line( + h, "WWW-Authenticate: Bearer realm=\"https://r/token\",service=\"r\"\r\n"); + util::curl::parse_header_line( + h, "Location: https://r/v2/x/blobs/uploads/abc123\r\n"); + util::curl::parse_header_line( + h, "Docker-Content-Digest: sha256:deadbeef\r\n"); + + REQUIRE(h.get("www-authenticate") == + "Bearer realm=\"https://r/token\",service=\"r\""); + REQUIRE(h.get("location") == "https://r/v2/x/blobs/uploads/abc123"); + REQUIRE(h.get("docker-content-digest") == "sha256:deadbeef"); +} + +TEST_CASE("curl parse_header_line edge cases", "[curl]") { + util::curl::headers h; + + // status line and blank separator carry no colon -> ignored + util::curl::parse_header_line(h, "HTTP/1.1 200 OK\r\n"); + util::curl::parse_header_line(h, "\r\n"); + REQUIRE(h.entries.empty()); + + // surrounding whitespace is trimmed from name and value; a value may itself + // contain a colon (e.g. a URL or digest). + util::curl::parse_header_line(h, " Location : https://host:443/path \r\n"); + REQUIRE(h.get("location") == "https://host:443/path"); + + // empty value is preserved + util::curl::parse_header_line(h, "X-Empty:\r\n"); + REQUIRE(h.get("x-empty") == ""); +} From cc31133c6fb18bded86bee4cadf031b1207fec95 Mon Sep 17 00:00:00 2001 From: bcumming Date: Mon, 29 Jun 2026 16:31:23 +0200 Subject: [PATCH 03/29] basic oci support - still using oras --- meson.build | 2 + src/uenv/oci/auth.cpp | 269 +++++++++++++++++++++++++++ src/uenv/oci/auth.h | 80 ++++++++ src/uenv/oci/client.cpp | 385 +++++++++++++++++++++++++++++++++++++++ src/uenv/oci/client.h | 142 +++++++++++++++ test/meson.build | 2 + test/unit/oci_auth.cpp | 97 ++++++++++ test/unit/oci_client.cpp | 92 ++++++++++ 8 files changed, 1069 insertions(+) create mode 100644 src/uenv/oci/auth.cpp create mode 100644 src/uenv/oci/auth.h create mode 100644 src/uenv/oci/client.cpp create mode 100644 src/uenv/oci/client.h create mode 100644 test/unit/oci_auth.cpp create mode 100644 test/unit/oci_client.cpp diff --git a/meson.build b/meson.build index 54e713ed..196b4a61 100644 --- a/meson.build +++ b/meson.build @@ -62,6 +62,8 @@ lib_src = [ 'src/uenv/log.cpp', 'src/uenv/meta.cpp', 'src/uenv/mount.cpp', + 'src/uenv/oci/auth.cpp', + 'src/uenv/oci/client.cpp', 'src/uenv/oras.cpp', 'src/uenv/parse.cpp', 'src/uenv/print.cpp', diff --git a/src/uenv/oci/auth.cpp b/src/uenv/oci/auth.cpp new file mode 100644 index 00000000..8122f3b2 --- /dev/null +++ b/src/uenv/oci/auth.cpp @@ -0,0 +1,269 @@ +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +#include "auth.h" + +namespace uenv { +namespace oci { + +namespace { + +std::string to_lower(std::string_view s) { + std::string out; + out.reserve(s.size()); + for (char c : s) { + out.push_back( + static_cast(std::tolower(static_cast(c)))); + } + return out; +} + +std::string_view trim(std::string_view s) { + auto is_space = [](char c) { + return std::isspace(static_cast(c)) != 0; + }; + while (!s.empty() && is_space(s.front())) { + s.remove_prefix(1); + } + while (!s.empty() && is_space(s.back())) { + s.remove_suffix(1); + } + return s; +} + +// split a scope value on spaces (the OCI spec permits a single scope parameter +// carrying a space-separated list). +void append_scopes(std::vector& out, std::string_view value) { + std::size_t i = 0; + while (i < value.size()) { + while (i < value.size() && + std::isspace(static_cast(value[i]))) { + ++i; + } + std::size_t start = i; + while (i < value.size() && + !std::isspace(static_cast(value[i]))) { + ++i; + } + if (i > start) { + out.emplace_back(value.substr(start, i - start)); + } + } +} + +} // namespace + +std::optional +parse_bearer_challenge(std::string_view v) { + std::string_view s = trim(v); + + // case-insensitive "Bearer" scheme, followed by whitespace + constexpr std::string_view scheme = "bearer"; + if (s.size() < scheme.size()) { + return std::nullopt; + } + for (std::size_t i = 0; i < scheme.size(); ++i) { + if (std::tolower(static_cast(s[i])) != scheme[i]) { + return std::nullopt; + } + } + s.remove_prefix(scheme.size()); + if (!s.empty() && !std::isspace(static_cast(s.front()))) { + return std::nullopt; // e.g. "BearerX" is not the Bearer scheme + } + s = trim(s); + + bearer_challenge challenge; + + std::size_t i = 0; + while (i < s.size()) { + // key = up to '=' or ',' + std::size_t key_start = i; + while (i < s.size() && s[i] != '=' && s[i] != ',') { + ++i; + } + std::string key = to_lower(trim(s.substr(key_start, i - key_start))); + + std::string value; + if (i < s.size() && s[i] == '=') { + ++i; // consume '=' + if (i < s.size() && s[i] == '"') { + ++i; // consume opening quote + std::string buf; + while (i < s.size() && s[i] != '"') { + if (s[i] == '\\' && i + 1 < s.size()) { + buf.push_back(s[i + 1]); + i += 2; + } else { + buf.push_back(s[i]); + ++i; + } + } + if (i < s.size() && s[i] == '"') { + ++i; // consume closing quote + } + value = std::move(buf); + } else { + std::size_t val_start = i; + while (i < s.size() && s[i] != ',') { + ++i; + } + value = std::string(trim(s.substr(val_start, i - val_start))); + } + } + + if (key == "realm") { + challenge.realm = std::move(value); + } else if (key == "service") { + challenge.service = std::move(value); + } else if (key == "scope") { + append_scopes(challenge.scopes, value); + } + + // advance past the separating comma and any whitespace + while (i < s.size() && s[i] != ',') { + ++i; + } + if (i < s.size() && s[i] == ',') { + ++i; + } + s = trim(s.substr(i)); + i = 0; + } + + if (challenge.realm.empty()) { + return std::nullopt; // realm is mandatory + } + return challenge; +} + +std::string token_url(const bearer_challenge& challenge, + const std::vector& scopes) { + std::string url = challenge.realm; + char sep = url.find('?') == std::string::npos ? '?' : '&'; + if (!challenge.service.empty()) { + url += sep; + url += "service="; + url += challenge.service; + sep = '&'; + } + for (const auto& scope : scopes) { + url += sep; + url += "scope="; + url += scope; + sep = '&'; + } + return url; +} + +std::optional parse_token_response(std::string_view body) { + auto j = nlohmann::json::parse(body, nullptr, /*allow_exceptions=*/false); + if (j.is_discarded() || !j.is_object()) { + return std::nullopt; + } + if (auto it = j.find("token"); it != j.end() && it->is_string()) { + return it->get(); + } + if (auto it = j.find("access_token"); it != j.end() && it->is_string()) { + return it->get(); + } + return std::nullopt; +} + +std::string repository_scope(std::string_view repository, + std::string_view actions) { + return fmt::format("repository:{}:{}", repository, actions); +} + +util::expected +discover_challenge(const std::string& registry_url) { + // normalise: drop any trailing '/' before appending the v2 base path. + std::string base = registry_url; + while (!base.empty() && base.back() == '/') { + base.pop_back(); + } + + util::curl::request req; + req.url = base + "/v2/"; + spdlog::trace("oci::discover_challenge probing {}", req.url); + + auto resp = util::curl::perform(req); + if (!resp) { + return util::unexpected{fmt::format("failed to probe {}: {}", req.url, + resp.error().message)}; + } + if (resp->status == 200) { + return util::unexpected{fmt::format( + "{} permits anonymous access and issued no auth challenge", + req.url)}; + } + if (resp->status != 401) { + return util::unexpected{fmt::format( + "unexpected status {} probing {}", resp->status, req.url)}; + } + auto header = resp->headers.get("www-authenticate"); + if (!header) { + return util::unexpected{fmt::format( + "{} returned 401 with no WWW-Authenticate header", req.url)}; + } + auto challenge = parse_bearer_challenge(*header); + if (!challenge) { + return util::unexpected{fmt::format( + "could not parse WWW-Authenticate header: {}", *header)}; + } + return *challenge; +} + +util::expected +fetch_token(const bearer_challenge& challenge, + const std::vector& scopes, + const std::optional& creds) { + util::curl::request req; + req.url = token_url(challenge, scopes); + if (creds) { + req.username = creds->username; + req.password = creds->password; + } + spdlog::trace("oci::fetch_token requesting {}", req.url); + + auto resp = util::curl::perform(req); + if (!resp) { + return util::unexpected{fmt::format("token request failed: {}", + resp.error().message)}; + } + if (resp->status != 200) { + return util::unexpected{ + fmt::format("token request to {} failed with status {}: {}", + req.url, resp->status, resp->body)}; + } + auto token = parse_token_response(resp->body); + if (!token) { + return util::unexpected{ + "token endpoint response did not contain a token"}; + } + return *token; +} + +util::expected +authenticate(const std::string& registry_url, + const std::vector& scopes, + const std::optional& creds) { + auto challenge = discover_challenge(registry_url); + if (!challenge) { + return util::unexpected{challenge.error()}; + } + return fetch_token(*challenge, scopes, creds); +} + +} // namespace oci +} // namespace uenv diff --git a/src/uenv/oci/auth.h b/src/uenv/oci/auth.h new file mode 100644 index 00000000..a9f32c5c --- /dev/null +++ b/src/uenv/oci/auth.h @@ -0,0 +1,80 @@ +#pragma once + +#include +#include +#include +#include + +#include + +namespace uenv { +namespace oci { + +// HTTP basic-auth credentials used to obtain a push/private-pull token. +// (defined here rather than reusing oras::credentials so the oci client does +// not depend on the oras module it is meant to replace.) +struct credentials { + std::string username; + std::string password; // password or personal access token +}; + +// A parsed `WWW-Authenticate: Bearer ...` challenge. `realm` is the token +// endpoint; `service` and `scopes` are the parameters to echo back when +// requesting a token. +struct bearer_challenge { + std::string realm; + std::string service; + std::vector scopes; + + friend bool operator==(const bearer_challenge&, + const bearer_challenge&) = default; +}; + +// --- pure helpers (no network; unit-tested) ----------------------------- + +// Parse the value of a `WWW-Authenticate` header. Returns nullopt if it is not +// a Bearer challenge or has no realm. Handles quoted values that themselves +// contain commas (e.g. scope="pull,push") and space-separated scope lists. +std::optional +parse_bearer_challenge(std::string_view header_value); + +// Build the token-request URL: ?service=&scope=&scope=. +// The scopes we intend to use are passed explicitly (the challenge's own scope +// is only advisory). OCI repository names and pull/push actions contain only +// query-safe characters, so values are placed verbatim. +std::string token_url(const bearer_challenge& challenge, + const std::vector& scopes); + +// Extract the bearer token from a token-endpoint JSON body, accepting both the +// `{"token":...}` and `{"access_token":...}` spellings. nullopt if neither is +// present or the body is not valid JSON. +std::optional parse_token_response(std::string_view body); + +// Compose an OCI auth scope string: repository:: +// e.g. repository_scope("deploy/todi/gh200/app/1.0", "pull,push"). +std::string repository_scope(std::string_view repository, + std::string_view actions); + +// --- network operations ------------------------------------------------- + +// Probe `/v2/` and parse the Bearer challenge from the 401. +util::expected +discover_challenge(const std::string& registry_url); + +// Request a bearer token for the given scopes, optionally authenticating with +// basic-auth credentials (required for push or private pull; omit for anonymous +// pull). Returns the raw token string for use as `Authorization: Bearer `. +util::expected +fetch_token(const bearer_challenge& challenge, + const std::vector& scopes, + const std::optional& creds = std::nullopt); + +// Convenience: discover the challenge for a registry and fetch a token in one +// call. +util::expected +authenticate(const std::string& registry_url, + const std::vector& scopes, + const std::optional& creds = std::nullopt); + +} // namespace oci +} // namespace uenv diff --git a/src/uenv/oci/client.cpp b/src/uenv/oci/client.cpp new file mode 100644 index 00000000..b04de7fc --- /dev/null +++ b/src/uenv/oci/client.cpp @@ -0,0 +1,385 @@ +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +#include "client.h" + +namespace uenv { +namespace oci { + +// --- pure helpers ------------------------------------------------------- + +std::string digest_string(const util::sha256_digest& d) { + return "sha256:" + d.hex(); +} + +std::string blob_path(std::string_view repository, std::string_view digest) { + return fmt::format("/v2/{}/blobs/{}", repository, digest); +} + +std::string manifest_path(std::string_view repository, + std::string_view reference) { + return fmt::format("/v2/{}/manifests/{}", repository, reference); +} + +std::string uploads_path(std::string_view repository) { + return fmt::format("/v2/{}/blobs/uploads/", repository); +} + +std::string tags_path(std::string_view repository) { + return fmt::format("/v2/{}/tags/list", repository); +} + +std::string referrers_path(std::string_view repository, + std::string_view digest) { + return fmt::format("/v2/{}/referrers/{}", repository, digest); +} + +std::string resolve_upload_url(std::string_view registry_url, + std::string_view location, + std::string_view digest) { + std::string url; + // the Location may be absolute (https://...) or registry-relative (/v2/...). + if (location.rfind("http://", 0) == 0 || + location.rfind("https://", 0) == 0) { + url = std::string{location}; + } else { + std::string base{registry_url}; + while (!base.empty() && base.back() == '/') { + base.pop_back(); + } + if (!location.empty() && location.front() != '/') { + base.push_back('/'); + } + url = base + std::string{location}; + } + // append the digest query parameter the monolithic PUT requires. + url += (url.find('?') == std::string::npos) ? '?' : '&'; + url += "digest="; + url += digest; + return url; +} + +std::optional> +parse_tags_list(std::string_view body) { + auto j = nlohmann::json::parse(body, nullptr, /*allow_exceptions=*/false); + if (j.is_discarded() || !j.is_object()) { + return std::nullopt; + } + std::vector tags; + if (auto it = j.find("tags"); it != j.end() && it->is_array()) { + for (const auto& t : *it) { + if (t.is_string()) { + tags.push_back(t.get()); + } + } + } + return tags; +} + +std::optional> parse_referrers(std::string_view body) { + auto j = nlohmann::json::parse(body, nullptr, /*allow_exceptions=*/false); + if (j.is_discarded() || !j.is_object()) { + return std::nullopt; + } + std::vector out; + auto it = j.find("manifests"); + if (it == j.end() || !it->is_array()) { + return out; + } + for (const auto& m : *it) { + if (!m.is_object()) { + continue; + } + descriptor d; + d.media_type = m.value("mediaType", ""); + d.digest = m.value("digest", ""); + d.size = m.value("size", std::size_t{0}); + if (auto a = m.find("artifactType"); + a != m.end() && a->is_string()) { + d.artifact_type = a->get(); + } + out.push_back(std::move(d)); + } + return out; +} + +// --- client ------------------------------------------------------------- + +util::expected +client::create(std::string registry_url, std::string repository, + std::optional creds) { + auto challenge = discover_challenge(registry_url); + if (!challenge) { + return util::unexpected{challenge.error()}; + } + + client c; + c.registry_url_ = std::move(registry_url); + while (!c.registry_url_.empty() && c.registry_url_.back() == '/') { + c.registry_url_.pop_back(); + } + c.repository_ = std::move(repository); + c.creds_ = std::move(creds); + c.challenge_ = std::move(*challenge); + return c; +} + +util::expected client::token_for(bool write) { + auto& cache = write ? push_token_ : pull_token_; + if (cache) { + return *cache; + } + auto scope = repository_scope(repository_, write ? "pull,push" : "pull"); + auto token = fetch_token(challenge_, {scope}, creds_); + if (!token) { + return util::unexpected{token.error()}; + } + cache = *token; + return *cache; +} + +util::expected +client::blob_exists(const std::string& digest) { + auto token = token_for(false); + if (!token) { + return util::unexpected{token.error()}; + } + util::curl::request req; + req.url = registry_url_ + blob_path(repository_, digest); + req.method = util::curl::http_method::head; + req.bearer_token = *token; + + auto resp = util::curl::perform(req); + if (!resp) { + return util::unexpected{resp.error().message}; + } + if (resp->status == 200) { + return true; + } + if (resp->status == 404) { + return false; + } + return util::unexpected{fmt::format( + "unexpected status {} for HEAD {}", resp->status, digest)}; +} + +util::expected +client::get_blob(const std::string& digest) { + auto token = token_for(false); + if (!token) { + return util::unexpected{token.error()}; + } + util::curl::request req; + req.url = registry_url_ + blob_path(repository_, digest); + req.bearer_token = *token; + // registries 307-redirect blob downloads to backing storage. + req.follow_redirects = true; + + auto resp = util::curl::perform(req); + if (!resp) { + return util::unexpected{resp.error().message}; + } + if (resp->status != 200) { + return util::unexpected{fmt::format( + "failed to fetch blob {} (status {})", digest, resp->status)}; + } + return resp->body; +} + +util::expected +client::get_manifest(const std::string& reference) { + auto token = token_for(false); + if (!token) { + return util::unexpected{token.error()}; + } + util::curl::request req; + req.url = registry_url_ + manifest_path(repository_, reference); + req.bearer_token = *token; + req.header_lines = {fmt::format("Accept: {}", media_type_manifest), + fmt::format("Accept: {}", media_type_index)}; + + auto resp = util::curl::perform(req); + if (!resp) { + return util::unexpected{resp.error().message}; + } + if (resp->status != 200) { + return util::unexpected{fmt::format( + "failed to fetch manifest {} (status {})", reference, + resp->status)}; + } + manifest_response m; + m.body = std::move(resp->body); + m.digest = resp->headers.get("docker-content-digest").value_or(""); + m.media_type = resp->headers.get("content-type").value_or(""); + return m; +} + +util::expected, std::string> client::list_tags() { + auto token = token_for(false); + if (!token) { + return util::unexpected{token.error()}; + } + util::curl::request req; + req.url = registry_url_ + tags_path(repository_); + req.bearer_token = *token; + + auto resp = util::curl::perform(req); + if (!resp) { + return util::unexpected{resp.error().message}; + } + if (resp->status != 200) { + return util::unexpected{fmt::format( + "failed to list tags (status {})", resp->status)}; + } + auto tags = parse_tags_list(resp->body); + if (!tags) { + return util::unexpected{"could not parse tags/list response"}; + } + return *tags; +} + +util::expected, std::string> +client::referrers(const std::string& digest) { + auto token = token_for(false); + if (!token) { + return util::unexpected{token.error()}; + } + util::curl::request req; + req.url = registry_url_ + referrers_path(repository_, digest); + req.bearer_token = *token; + + auto resp = util::curl::perform(req); + if (!resp) { + return util::unexpected{resp.error().message}; + } + if (resp->status != 200) { + return util::unexpected{fmt::format( + "failed to fetch referrers of {} (status {})", digest, + resp->status)}; + } + auto refs = parse_referrers(resp->body); + if (!refs) { + return util::unexpected{"could not parse referrers response"}; + } + return *refs; +} + +namespace { + +// drive the monolithic upload handshake: POST an upload session, then PUT the +// payload (carried by `body_setup`) to the returned Location with ?digest=. +util::expected +do_put_blob(const std::string& registry_url, const std::string& uploads, + const std::string& token, const std::string& digest, + const std::function& body_setup) { + // 1. open an upload session + util::curl::request post; + post.url = registry_url + uploads; + post.method = util::curl::http_method::post; + post.bearer_token = token; + auto opened = util::curl::perform(post); + if (!opened) { + return util::unexpected{opened.error().message}; + } + if (opened->status != 202) { + return util::unexpected{fmt::format( + "failed to open upload session (status {}): {}", opened->status, + opened->body)}; + } + auto location = opened->headers.get("location"); + if (!location) { + return util::unexpected{ + "upload session response had no Location header"}; + } + + // 2. PUT the payload to ?digest= + util::curl::request put; + put.url = resolve_upload_url(registry_url, *location, digest); + put.method = util::curl::http_method::put; + put.bearer_token = token; + put.header_lines = { + fmt::format("Content-Type: {}", media_type_octet_stream)}; + body_setup(put); + + auto done = util::curl::perform(put); + if (!done) { + return util::unexpected{done.error().message}; + } + if (done->status != 201) { + return util::unexpected{fmt::format( + "failed to upload blob {} (status {}): {}", digest, done->status, + done->body)}; + } + return {}; +} + +} // namespace + +util::expected +client::put_blob(const std::string& digest, + const std::filesystem::path& file) { + if (auto exists = blob_exists(digest); exists && *exists) { + spdlog::trace("oci::put_blob {} already present", digest); + return {}; + } + auto token = token_for(true); + if (!token) { + return util::unexpected{token.error()}; + } + return do_put_blob(registry_url_, uploads_path(repository_), *token, digest, + [&file](util::curl::request& r) { + r.upload_file = file; + }); +} + +util::expected +client::put_blob_bytes(const std::string& digest, const std::string& data) { + if (auto exists = blob_exists(digest); exists && *exists) { + return {}; + } + auto token = token_for(true); + if (!token) { + return util::unexpected{token.error()}; + } + return do_put_blob(registry_url_, uploads_path(repository_), *token, digest, + [&data](util::curl::request& r) { r.body = data; }); +} + +util::expected +client::put_manifest(const std::string& reference, const std::string& body, + std::string_view media_type) { + auto token = token_for(true); + if (!token) { + return util::unexpected{token.error()}; + } + util::curl::request req; + req.url = registry_url_ + manifest_path(repository_, reference); + req.method = util::curl::http_method::put; + req.bearer_token = *token; + req.body = body; + req.header_lines = {fmt::format("Content-Type: {}", media_type)}; + + auto resp = util::curl::perform(req); + if (!resp) { + return util::unexpected{resp.error().message}; + } + if (resp->status != 201) { + return util::unexpected{fmt::format( + "failed to put manifest {} (status {}): {}", reference, + resp->status, resp->body)}; + } + return resp->headers.get("docker-content-digest").value_or(""); +} + +} // namespace oci +} // namespace uenv diff --git a/src/uenv/oci/client.h b/src/uenv/oci/client.h new file mode 100644 index 00000000..3a0ce8f7 --- /dev/null +++ b/src/uenv/oci/client.h @@ -0,0 +1,142 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace uenv { +namespace oci { + +// common OCI media types +inline constexpr std::string_view media_type_manifest = + "application/vnd.oci.image.manifest.v1+json"; +inline constexpr std::string_view media_type_index = + "application/vnd.oci.image.index.v1+json"; +inline constexpr std::string_view media_type_octet_stream = + "application/octet-stream"; + +// An OCI content descriptor (a manifest entry / referrer record). +struct descriptor { + std::string media_type; + std::string digest; // "sha256:" + std::size_t size = 0; + std::optional artifact_type; + + friend bool operator==(const descriptor&, const descriptor&) = default; +}; + +// The result of fetching a manifest: the raw bytes plus the registry-reported +// digest and media type. The bytes are what must be re-digested locally to +// confirm identity. +struct manifest_response { + std::string body; + std::string digest; // value of the Docker-Content-Digest header + std::string media_type; +}; + +// --- pure helpers (no network; unit-tested) ----------------------------- + +// "sha256:" for a digest. +std::string digest_string(const util::sha256_digest& d); + +// OCI Distribution v2 path builders (each begins with "/v2/"). +std::string blob_path(std::string_view repository, std::string_view digest); +std::string manifest_path(std::string_view repository, + std::string_view reference); +std::string uploads_path(std::string_view repository); +std::string tags_path(std::string_view repository); +std::string referrers_path(std::string_view repository, + std::string_view digest); + +// Resolve a blob-upload Location (which may be absolute or registry-relative) +// against the registry base URL, then append the monolithic-upload digest query +// parameter, choosing '?' or '&' as appropriate. +std::string resolve_upload_url(std::string_view registry_url, + std::string_view location, + std::string_view digest); + +// Parse a /tags/list body ({"name":...,"tags":[...]}) into the tag list. +std::optional> +parse_tags_list(std::string_view body); + +// Parse a referrers index body into its manifest descriptors. +std::optional> parse_referrers(std::string_view body); + +// --- registry client ---------------------------------------------------- + +// A client bound to one repository on one registry. Authenticates lazily, +// caching a pull token and (separately) a pull,push token as operations need +// them. Read operations work anonymously when no credentials are supplied. +class client { + public: + // Probe the registry, parse its auth challenge, and bind to `repository` + // (e.g. "deploy/todi/gh200/app/1.0"). Supply credentials for push or + // private pull; omit for anonymous pull. + static util::expected + create(std::string registry_url, std::string repository, + std::optional creds = std::nullopt); + + // does a blob exist? HEAD; 200 -> true, 404 -> false. + util::expected blob_exists(const std::string& digest); + + // fetch a blob's bytes (follows the 307 redirect to backing storage). + util::expected + get_blob(const std::string& digest); + + // fetch a manifest by tag or digest. + util::expected + get_manifest(const std::string& reference); + + // list the repository's tags. + util::expected, std::string> list_tags(); + + // list artifacts that refer to `digest` (replaces `oras discover`). + util::expected, std::string> + referrers(const std::string& digest); + + // upload a blob via the monolithic POST-then-PUT handshake. the file is + // streamed from disk (not held in memory), so large squashfs layers are + // fine. no-op if the blob already exists. + util::expected + put_blob(const std::string& digest, const std::filesystem::path& file); + + // upload an in-memory blob (config blobs, small payloads). + util::expected + put_blob_bytes(const std::string& digest, const std::string& data); + + // PUT a manifest under `reference` (tag or digest). returns the registry's + // Docker-Content-Digest (the canonical id). + util::expected + put_manifest(const std::string& reference, const std::string& body, + std::string_view media_type = media_type_manifest); + + const std::string& registry_url() const { + return registry_url_; + } + const std::string& repository() const { + return repository_; + } + + private: + client() = default; + + // return a bearer token for this repository, fetching/caching as needed. + util::expected token_for(bool write); + + std::string registry_url_; // normalised, no trailing '/' + std::string repository_; + std::optional creds_; + bearer_challenge challenge_; + std::optional pull_token_; + std::optional push_token_; +}; + +} // namespace oci +} // namespace uenv diff --git a/test/meson.build b/test/meson.build index a993a2df..61af0eba 100644 --- a/test/meson.build +++ b/test/meson.build @@ -69,6 +69,8 @@ unit_src = [ 'unit/lex.cpp', 'unit/main.cpp', 'unit/mount.cpp', + 'unit/oci_auth.cpp', + 'unit/oci_client.cpp', 'unit/parse.cpp', 'unit/shell.cpp', 'unit/signal.cpp', diff --git a/test/unit/oci_auth.cpp b/test/unit/oci_auth.cpp new file mode 100644 index 00000000..3aebe10b --- /dev/null +++ b/test/unit/oci_auth.cpp @@ -0,0 +1,97 @@ +#include + +#include + +using uenv::oci::parse_bearer_challenge; +using uenv::oci::parse_token_response; +using uenv::oci::repository_scope; +using uenv::oci::token_url; + +TEST_CASE("oci parse_bearer_challenge from jfrog", "[oci][auth]") { + // the exact challenge the CSCS jfrog registry returns (spike step 1) + auto c = parse_bearer_challenge( + "Bearer realm=\"https://jfrog.svc.cscs.ch/v2/token\"," + "service=\"jfrog.svc.cscs.ch\""); + REQUIRE(c.has_value()); + REQUIRE(c->realm == "https://jfrog.svc.cscs.ch/v2/token"); + REQUIRE(c->service == "jfrog.svc.cscs.ch"); + REQUIRE(c->scopes.empty()); +} + +TEST_CASE("oci parse_bearer_challenge with scope containing a comma", + "[oci][auth]") { + // the scope value itself contains a comma (pull,push) inside quotes - a + // naive comma split would break here. + auto c = parse_bearer_challenge( + "Bearer realm=\"https://r/token\",service=\"reg\"," + "scope=\"repository:foo/bar:pull,push\""); + REQUIRE(c.has_value()); + REQUIRE(c->realm == "https://r/token"); + REQUIRE(c->service == "reg"); + REQUIRE(c->scopes == std::vector{"repository:foo/bar:pull,push"}); +} + +TEST_CASE("oci parse_bearer_challenge space-separated scopes", "[oci][auth]") { + auto c = parse_bearer_challenge( + "Bearer realm=\"https://r/token\",service=\"reg\"," + "scope=\"repository:a:pull repository:b:pull\""); + REQUIRE(c.has_value()); + REQUIRE(c->scopes == std::vector{"repository:a:pull", + "repository:b:pull"}); +} + +TEST_CASE("oci parse_bearer_challenge rejects non-bearer / no realm", + "[oci][auth]") { + REQUIRE_FALSE(parse_bearer_challenge("Basic realm=\"r\"").has_value()); + REQUIRE_FALSE(parse_bearer_challenge("Bearerish realm=\"r\"").has_value()); + REQUIRE_FALSE(parse_bearer_challenge("").has_value()); + // a Bearer challenge without a realm is unusable + REQUIRE_FALSE( + parse_bearer_challenge("Bearer service=\"reg\"").has_value()); +} + +TEST_CASE("oci parse_bearer_challenge is scheme-case-insensitive", + "[oci][auth]") { + auto c = parse_bearer_challenge("bearer realm=\"https://r/token\""); + REQUIRE(c.has_value()); + REQUIRE(c->realm == "https://r/token"); +} + +TEST_CASE("oci token_url", "[oci][auth]") { + uenv::oci::bearer_challenge c{ + .realm = "https://jfrog.svc.cscs.ch/v2/token", + .service = "jfrog.svc.cscs.ch", + .scopes = {}}; + + // matches the spike's proven-working token request URL + REQUIRE(token_url(c, {"repository:uenv/deploy/x/24.7:pull"}) == + "https://jfrog.svc.cscs.ch/v2/token?service=jfrog.svc.cscs.ch" + "&scope=repository:uenv/deploy/x/24.7:pull"); + + // multiple scopes become repeated scope= parameters + REQUIRE(token_url(c, {"repository:a:pull", "repository:b:push"}) == + "https://jfrog.svc.cscs.ch/v2/token?service=jfrog.svc.cscs.ch" + "&scope=repository:a:pull&scope=repository:b:push"); + + // a realm that already carries a query continues with '&' + uenv::oci::bearer_challenge q{ + .realm = "https://r/token?foo=bar", .service = "reg", .scopes = {}}; + REQUIRE(token_url(q, {}) == "https://r/token?foo=bar&service=reg"); +} + +TEST_CASE("oci parse_token_response", "[oci][auth]") { + REQUIRE(parse_token_response(R"({"token":"abc123"})") == "abc123"); + REQUIRE(parse_token_response(R"({"access_token":"xyz"})") == "xyz"); + // token preferred when both present + REQUIRE(parse_token_response(R"({"token":"t","access_token":"a"})") == "t"); + // missing / malformed + REQUIRE(parse_token_response(R"({"expires_in":300})") == std::nullopt); + REQUIRE(parse_token_response("not json") == std::nullopt); + REQUIRE(parse_token_response("") == std::nullopt); +} + +TEST_CASE("oci repository_scope", "[oci][auth]") { + REQUIRE(repository_scope("deploy/todi/gh200/app/1.0", "pull") == + "repository:deploy/todi/gh200/app/1.0:pull"); + REQUIRE(repository_scope("foo", "pull,push") == "repository:foo:pull,push"); +} diff --git a/test/unit/oci_client.cpp b/test/unit/oci_client.cpp new file mode 100644 index 00000000..d65bb1ef --- /dev/null +++ b/test/unit/oci_client.cpp @@ -0,0 +1,92 @@ +#include + +#include +#include + +using namespace uenv::oci; + +TEST_CASE("oci digest_string", "[oci][client]") { + auto d = util::sha256_string("abc"); + REQUIRE(digest_string(d) == + "sha256:" + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); +} + +TEST_CASE("oci path builders", "[oci][client]") { + const std::string repo = "deploy/todi/gh200/app/1.0"; + REQUIRE(blob_path(repo, "sha256:abc") == + "/v2/deploy/todi/gh200/app/1.0/blobs/sha256:abc"); + REQUIRE(manifest_path(repo, "v1") == + "/v2/deploy/todi/gh200/app/1.0/manifests/v1"); + REQUIRE(uploads_path(repo) == + "/v2/deploy/todi/gh200/app/1.0/blobs/uploads/"); + REQUIRE(tags_path(repo) == "/v2/deploy/todi/gh200/app/1.0/tags/list"); + REQUIRE(referrers_path(repo, "sha256:abc") == + "/v2/deploy/todi/gh200/app/1.0/referrers/sha256:abc"); +} + +TEST_CASE("oci resolve_upload_url", "[oci][client]") { + const std::string reg = "https://jfrog.svc.cscs.ch"; + const std::string dig = "sha256:deadbeef"; + + // registry-relative Location, no existing query -> '?' + REQUIRE(resolve_upload_url(reg, "/v2/r/blobs/uploads/abc", dig) == + "https://jfrog.svc.cscs.ch/v2/r/blobs/uploads/abc?digest=sha256:" + "deadbeef"); + + // relative Location that already carries a query -> '&' + REQUIRE(resolve_upload_url(reg, "/v2/r/blobs/uploads/abc?_state=xyz", dig) == + "https://jfrog.svc.cscs.ch/v2/r/blobs/uploads/abc?_state=xyz&digest=" + "sha256:deadbeef"); + + // absolute Location (e.g. redirected to storage) is used verbatim + REQUIRE(resolve_upload_url(reg, "https://store.example/up/abc?sig=1", dig) == + "https://store.example/up/abc?sig=1&digest=sha256:deadbeef"); + + // trailing slash on the registry base is not doubled + REQUIRE(resolve_upload_url("https://reg/", "/v2/x", dig) == + "https://reg/v2/x?digest=sha256:deadbeef"); +} + +TEST_CASE("oci parse_tags_list", "[oci][client]") { + auto tags = + parse_tags_list(R"({"name":"r","tags":["v1","v2","latest"]})"); + REQUIRE(tags.has_value()); + REQUIRE(*tags == std::vector{"v1", "v2", "latest"}); + + // empty/absent tag list is valid (returns empty, not an error) + auto none = parse_tags_list(R"({"name":"r","tags":null})"); + REQUIRE(none.has_value()); + REQUIRE(none->empty()); + + REQUIRE(parse_tags_list("not json") == std::nullopt); +} + +TEST_CASE("oci parse_referrers", "[oci][client]") { + // an image-index body, as returned by /v2//referrers/ + const auto body = R"({ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:f7f04f3b", + "size": 1234, + "artifactType": "application/vnd.cscs.uenv.meta" + } + ] + })"; + auto refs = parse_referrers(body); + REQUIRE(refs.has_value()); + REQUIRE(refs->size() == 1); + REQUIRE((*refs)[0].digest == "sha256:f7f04f3b"); + REQUIRE((*refs)[0].size == 1234); + REQUIRE((*refs)[0].artifact_type == "application/vnd.cscs.uenv.meta"); + + // no manifests array -> empty list + auto empty = parse_referrers(R"({"manifests":[]})"); + REQUIRE(empty.has_value()); + REQUIRE(empty->empty()); + + REQUIRE(parse_referrers("not json") == std::nullopt); +} From f2e45c29fd9802edfdb137d402375722aae0ea86 Mon Sep 17 00:00:00 2001 From: bcumming Date: Tue, 30 Jun 2026 13:10:22 +0200 Subject: [PATCH 04/29] replace oras for image pull --- meson.build | 1 + src/cli/pull.cpp | 144 +++++++++++++++++++++-------- src/uenv/oci/client.cpp | 28 ++++++ src/uenv/oci/client.h | 12 +++ src/uenv/oci/pull.cpp | 195 ++++++++++++++++++++++++++++++++++++++++ src/uenv/oci/pull.h | 35 ++++++++ src/util/curl.cpp | 54 ++++++++++- src/util/curl.h | 15 ++++ 8 files changed, 443 insertions(+), 41 deletions(-) create mode 100644 src/uenv/oci/pull.cpp create mode 100644 src/uenv/oci/pull.h diff --git a/meson.build b/meson.build index 196b4a61..ceec3830 100644 --- a/meson.build +++ b/meson.build @@ -64,6 +64,7 @@ lib_src = [ 'src/uenv/mount.cpp', 'src/uenv/oci/auth.cpp', 'src/uenv/oci/client.cpp', + 'src/uenv/oci/pull.cpp', 'src/uenv/oras.cpp', 'src/uenv/parse.cpp', 'src/uenv/print.cpp', diff --git a/src/cli/pull.cpp b/src/cli/pull.cpp index 4eedf7de..bc8bdeec 100644 --- a/src/cli/pull.cpp +++ b/src/cli/pull.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -11,10 +12,13 @@ #include #include +#include +#include #include #include #include #include +#include #include #include #include @@ -172,26 +176,70 @@ int image_pull(const image_pull_args& args, const global_settings& settings) { spdlog::debug("pull sqfs: {}", pull_sqfs); if (pull_sqfs || pull_meta) { + // split the configured registry "host/prefix" (e.g. + // "jfrog.svc.cscs.ch/uenv") into a base URL and the repository prefix, + // then build the OCI repository name the same way the oras address was + // formed: /////. + std::string reg = registry_cfg.url; + for (const std::string scheme : {"https://", "http://"}) { + if (reg.rfind(scheme, 0) == 0) { + reg = reg.substr(scheme.size()); + break; + } + } + std::string host = reg; + std::string prefix; + if (auto slash = reg.find('/'); slash != std::string::npos) { + host = reg.substr(0, slash); + prefix = reg.substr(slash + 1); + } + const std::string registry_base = "https://" + host; + const std::string repository = + prefix.empty() + ? fmt::format("{}/{}/{}/{}/{}", nspace, record.system, + record.uarch, record.name, record.version) + : fmt::format("{}/{}/{}/{}/{}/{}", prefix, nspace, + record.system, record.uarch, record.name, + record.version); + + std::optional oci_creds; + if (credentials) { + oci_creds = + oci::credentials{credentials->username, credentials->token}; + } + + spdlog::debug("oci pull: registry={} repository={}", registry_base, + repository); + + auto client = oci::client::create(registry_base, repository, oci_creds); + if (!client) { + term::error("unable to connect to the registry:\n{}", + client.error()); + return 1; + } + + // identify the image by its manifest digest (record.sha). + const std::string manifest_ref = "sha256:" + record.sha.string(); + try { - auto rego_url = registry_cfg.url; - spdlog::debug("registry url: {}", rego_url); + // the image manifest is needed for the squashfs layer; fetch once. + auto manifest = client->get_manifest(manifest_ref); + if (!manifest) { + term::error("unable to fetch the image manifest:\n{}", + manifest.error()); + return 1; + } if (pull_meta) { - // the digests returned by oras::discover is a list of artifacts - // that have been "oras attach"ed to our squashfs image. This - // would be empty if no meta data was attached. - auto digests = - oras::discover(rego_url, nspace, record, credentials); - if (!digests) { - term::error("unable to pull meta digest.\n{}", - digests.error().message); + auto found = + oci::pull_meta(*client, manifest_ref, paths.store); + if (!found) { + term::error("unable to pull meta data.\n{}", found.error()); return 1; } - if (digests->empty()) { - // No metadata attached in the registry. - // If the user explicitly requested metadata (--only-meta or - // sqfs already exists), this is an error. Otherwise, warn - // and continue to pull the squashfs. + if (!*found) { + // No metadata attached. Error if the user explicitly wanted + // it; otherwise warn and continue to the squashfs. if (!pull_sqfs) { term::error("uenv exists in registry but has no " "attached metadata"); @@ -199,32 +247,54 @@ int image_pull(const image_pull_args& args, const global_settings& settings) { } term::warn( "uenv exists in registry but has no attached metadata"); - } else { - spdlog::debug("manifests: {}", fmt::join(*digests, ", ")); - - // We assume that there is one, and only, digest attached to - // the squashfs image: the meta data directory. - // pull_digest will download the digest: in the case of meta - // data it will unpack the meta path into paths.store. - const auto digest = *(digests->begin()); - - if (auto okay = - oras::pull_digest(rego_url, nspace, record, digest, - paths.store, credentials); - !okay) { - term::error("unable to pull uenv.\n{}", - okay.error().message); - return 1; - } } } if (pull_sqfs) { - auto tag_result = oras::pull_tag(rego_url, nspace, record, - paths.store, credentials); - if (!tag_result) { - term::error("unable to pull uenv.\n{}", - tag_result.error().message); + namespace bk = barkeep; + std::size_t downloaded_mb{0u}; + // round up so total_mb is never zero + std::size_t total_mb{(record.size_byte + (1024 * 1024 - 1)) / + (1024 * 1024)}; + auto bar = bk::ProgressBar( + &downloaded_mb, + { + .total = total_mb, + .message = fmt::format("pulling {}", record.id.string()), + .speed = 0.1, + .speed_unit = "MB/s", + .style = color::use_color() + ? bk::ProgressBarStyle::Rich + : bk::ProgressBarStyle::Bars, + .no_tty = !isatty(fileno(stdout)), + }); + + util::set_signal_catcher(); + auto progress = [&downloaded_mb](std::uint64_t now, + std::uint64_t) { + downloaded_mb = now / (1024 * 1024); + }; + // util::signal_raised() consumes (resets) the flag, so it must + // be checked exactly once. latch the result here in the abort + // predicate; the post-download check then reads the latch rather + // than calling signal_raised() again (which would see false and + // skip the cleanup, leaving a partial download behind). + bool aborted = false; + auto result = oci::pull_squashfs( + *client, *manifest, paths.store, progress, [&aborted]() { + aborted = aborted || util::signal_raised(); + return aborted; + }); + downloaded_mb = total_mb; + bar->done(); + + if (!result) { + // a Ctrl-C during the download aborts the transfer; surface + // it as a signal so the cleanup below runs. + if (aborted) { + throw util::signal_exception(util::last_signal_raised()); + } + term::error("unable to pull uenv.\n{}", result.error()); return 1; } } diff --git a/src/uenv/oci/client.cpp b/src/uenv/oci/client.cpp index b04de7fc..d40c355d 100644 --- a/src/uenv/oci/client.cpp +++ b/src/uenv/oci/client.cpp @@ -196,6 +196,34 @@ client::get_blob(const std::string& digest) { return resp->body; } +util::expected +client::get_blob_to_file( + const std::string& digest, const std::filesystem::path& file, + std::function progress, + std::function should_abort) { + auto token = token_for(false); + if (!token) { + return util::unexpected{token.error()}; + } + util::curl::request req; + req.url = registry_url_ + blob_path(repository_, digest); + req.bearer_token = *token; + req.follow_redirects = true; + req.download_file = file; + req.on_download_progress = std::move(progress); + req.should_abort = std::move(should_abort); + + auto resp = util::curl::perform(req); + if (!resp) { + return util::unexpected{resp.error().message}; + } + if (resp->status != 200) { + return util::unexpected{fmt::format( + "failed to fetch blob {} (status {})", digest, resp->status)}; + } + return {}; +} + util::expected client::get_manifest(const std::string& reference) { auto token = token_for(false); diff --git a/src/uenv/oci/client.h b/src/uenv/oci/client.h index 3a0ce8f7..a4491d34 100644 --- a/src/uenv/oci/client.h +++ b/src/uenv/oci/client.h @@ -1,7 +1,9 @@ #pragma once #include +#include #include +#include #include #include #include @@ -90,6 +92,16 @@ class client { util::expected get_blob(const std::string& digest); + // stream a blob straight to a file, never holding it in memory (for the + // multi-GB squashfs layer). follows the 307 redirect to backing storage. + // an optional progress callback receives (bytes_downloaded, bytes_total); + // an optional abort predicate, polled during transfer, cancels it. + util::expected + get_blob_to_file( + const std::string& digest, const std::filesystem::path& file, + std::function progress = {}, + std::function should_abort = {}); + // fetch a manifest by tag or digest. util::expected get_manifest(const std::string& reference); diff --git a/src/uenv/oci/pull.cpp b/src/uenv/oci/pull.cpp new file mode 100644 index 00000000..108b0d1b --- /dev/null +++ b/src/uenv/oci/pull.cpp @@ -0,0 +1,195 @@ +#include +#include +#include + +#include +#include +#include + +#include +#include +#include + +#include "client.h" +#include "pull.h" + +namespace uenv { +namespace oci { + +namespace fs = std::filesystem; + +namespace { + +// title annotation oras gives the squashfs layer. +constexpr std::string_view squashfs_title = "store.squashfs"; + +// extract a gzipped tar (held in memory) into `dest`, using the system tar. +util::expected extract_targz(const std::string& data, + const fs::path& dest) { + auto tmp = dest / ".uenv-meta.tar.gz"; + { + std::ofstream out(tmp, std::ios::binary | std::ios::trunc); + if (!out) { + return util::unexpected{ + fmt::format("unable to write temporary {}", tmp.string())}; + } + out.write(data.data(), static_cast(data.size())); + if (!out) { + return util::unexpected{ + fmt::format("error writing temporary {}", tmp.string())}; + } + } + + auto proc = util::run( + {"tar", "-xzf", tmp.string(), "-C", dest.string()}); + if (!proc) { + fs::remove(tmp); + return util::unexpected{ + fmt::format("unable to run tar to unpack meta: {}", proc.error())}; + } + auto rc = proc->wait(); + fs::remove(tmp); + if (rc != 0) { + return util::unexpected{ + fmt::format("tar failed to unpack meta (exit {})", rc)}; + } + return {}; +} + +} // namespace + +util::expected +pull_squashfs(client& c, const manifest_response& manifest, + const fs::path& store, progress_fn progress, + std::function should_abort) { + auto j = nlohmann::json::parse(manifest.body, nullptr, + /*allow_exceptions=*/false); + if (j.is_discarded() || !j.contains("layers")) { + return util::unexpected{"image manifest has no layers"}; + } + + // pick the squashfs layer: prefer the one titled "store.squashfs", else the + // sole layer. + std::string digest; + for (const auto& layer : j["layers"]) { + auto ann = layer.find("annotations"); + if (ann != layer.end()) { + auto title = ann->find("org.opencontainers.image.title"); + if (title != ann->end() && title->is_string() && + title->get() == squashfs_title) { + digest = layer.value("digest", ""); + break; + } + } + } + if (digest.empty() && j["layers"].size() == 1) { + digest = j["layers"][0].value("digest", ""); + } + if (digest.empty()) { + return util::unexpected{ + "could not find the store.squashfs layer in the manifest"}; + } + + if (!fs::exists(store)) { + std::error_code ec; + fs::create_directories(store, ec); + if (ec) { + return util::unexpected{fmt::format("unable to create {}: {}", + store.string(), ec.message())}; + } + } + + const fs::path dest = store / "store.squashfs"; + spdlog::debug("oci::pull_squashfs {} -> {}", digest, dest.string()); + if (auto ok = c.get_blob_to_file(digest, dest, std::move(progress), + std::move(should_abort)); + !ok) { + return util::unexpected{ok.error()}; + } + + // self-verify: the on-disk file must re-digest to the layer digest. + auto actual = util::sha256_file(dest); + if (!actual) { + return util::unexpected{actual.error()}; + } + auto actual_digest = digest_string(*actual); + if (actual_digest != digest) { + std::error_code ec; + fs::remove(dest, ec); + return util::unexpected{fmt::format( + "downloaded squashfs digest mismatch: expected {}, got {}", digest, + actual_digest)}; + } + spdlog::debug("oci::pull_squashfs verified {}", actual_digest); + return {}; +} + +util::expected +pull_meta(client& c, const std::string& manifest_digest, const fs::path& store) { + auto refs = c.referrers(manifest_digest); + if (!refs) { + return util::unexpected{refs.error()}; + } + + // find the uenv/meta referrer. + std::string meta_manifest_digest; + for (const auto& r : *refs) { + if (r.artifact_type == "uenv/meta") { + meta_manifest_digest = r.digest; + break; + } + } + if (meta_manifest_digest.empty()) { + return false; // no attached meta + } + + auto meta_manifest = c.get_manifest(meta_manifest_digest); + if (!meta_manifest) { + return util::unexpected{meta_manifest.error()}; + } + auto j = nlohmann::json::parse(meta_manifest->body, nullptr, + /*allow_exceptions=*/false); + if (j.is_discarded() || !j.contains("layers") || j["layers"].empty()) { + return util::unexpected{"meta manifest has no layers"}; + } + + // the meta layer is the gzipped tar marked for unpacking. + std::string digest; + for (const auto& layer : j["layers"]) { + auto ann = layer.find("annotations"); + if (ann != layer.end()) { + auto unpack = ann->find("io.deis.oras.content.unpack"); + if (unpack != ann->end() && unpack->is_string() && + unpack->get() == "true") { + digest = layer.value("digest", ""); + break; + } + } + } + if (digest.empty()) { + digest = j["layers"][0].value("digest", ""); + } + + spdlog::debug("oci::pull_meta layer {}", digest); + auto blob = c.get_blob(digest); + if (!blob) { + return util::unexpected{blob.error()}; + } + + if (!fs::exists(store)) { + std::error_code ec; + fs::create_directories(store, ec); + if (ec) { + return util::unexpected{fmt::format("unable to create {}: {}", + store.string(), ec.message())}; + } + } + + if (auto ok = extract_targz(*blob, store); !ok) { + return util::unexpected{ok.error()}; + } + return true; +} + +} // namespace oci +} // namespace uenv diff --git a/src/uenv/oci/pull.h b/src/uenv/oci/pull.h new file mode 100644 index 00000000..67d1ab92 --- /dev/null +++ b/src/uenv/oci/pull.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include + +namespace uenv { +namespace oci { + +// progress callback for the squashfs download: (bytes_downloaded, bytes_total). +using progress_fn = std::function; + +// Download the squashfs layer of `manifest` into /store.squashfs and +// self-verify: the file is re-digested and checked against the layer digest, so +// a truncated or corrupt download is caught locally. `manifest` is the image +// manifest already fetched via client::get_manifest. An optional abort +// predicate, polled during the download, cancels it (for Ctrl-C handling). +util::expected +pull_squashfs(client& c, const manifest_response& manifest, + const std::filesystem::path& store, progress_fn progress = {}, + std::function should_abort = {}); + +// Download and unpack the `uenv/meta` referrer (a gzipped tar) into , +// reproducing the `meta/` directory. Returns true if meta was found and +// pulled, false if the image has no attached meta. +util::expected +pull_meta(client& c, const std::string& manifest_digest, + const std::filesystem::path& store); + +} // namespace oci +} // namespace uenv diff --git a/src/util/curl.cpp b/src/util/curl.cpp index f76685f1..f45fd675 100644 --- a/src/util/curl.cpp +++ b/src/util/curl.cpp @@ -273,6 +273,25 @@ size_t header_callback(char* buffer, size_t size, size_t nitems, return total; } +size_t file_write_callback(void* source, size_t size, size_t n, void* target) { + const size_t realsize = size * n; + FILE* f = static_cast(target); + return fwrite(source, 1, realsize, f); +} + +int xferinfo_callback(void* p, curl_off_t dltotal, curl_off_t dlnow, + curl_off_t, curl_off_t) { + const auto& req = *static_cast(p); + if (req.on_download_progress) { + req.on_download_progress(static_cast(dlnow), + static_cast(dltotal)); + } + if (req.should_abort && req.should_abort()) { + return 1; // non-zero aborts the transfer (CURLE_ABORTED_BY_CALLBACK) + } + return 0; +} + const char* method_name(http_method m) { switch (m) { case http_method::get: @@ -385,11 +404,31 @@ expected perform(const request& req) { (long)req.body->size())); } - // capture body + // capture the response body: either streamed to a file (for large blob + // downloads) or buffered in memory. std::vector body; - body.reserve(200000); - CURL_EASY(curl_easy_setopt(h, CURLOPT_WRITEFUNCTION, memory_callback)); - CURL_EASY(curl_easy_setopt(h, CURLOPT_WRITEDATA, (void*)&body)); + FILE* download = nullptr; + auto cleanup_download = defer([&download]() { + if (download) { + fclose(download); + } + }); + if (req.download_file) { + download = fopen(req.download_file->c_str(), "wb"); + if (!download) { + return unexpected{ + error{CURLE_WRITE_ERROR, + fmt::format("unable to open {} for download", + req.download_file->string())}}; + } + CURL_EASY( + curl_easy_setopt(h, CURLOPT_WRITEFUNCTION, file_write_callback)); + CURL_EASY(curl_easy_setopt(h, CURLOPT_WRITEDATA, (void*)download)); + } else { + body.reserve(200000); + CURL_EASY(curl_easy_setopt(h, CURLOPT_WRITEFUNCTION, memory_callback)); + CURL_EASY(curl_easy_setopt(h, CURLOPT_WRITEDATA, (void*)&body)); + } // capture response headers response resp; @@ -400,6 +439,13 @@ expected perform(const request& req) { req.connect_timeout_ms)); CURL_EASY(curl_easy_setopt(h, CURLOPT_TIMEOUT_MS, req.timeout_ms)); + if (req.on_download_progress || req.should_abort) { + CURL_EASY(curl_easy_setopt(h, CURLOPT_NOPROGRESS, 0L)); + CURL_EASY( + curl_easy_setopt(h, CURLOPT_XFERINFOFUNCTION, xferinfo_callback)); + CURL_EASY(curl_easy_setopt(h, CURLOPT_XFERINFODATA, (void*)&req)); + } + CURL_EASY(curl_easy_perform(h)); CURL_EASY(curl_easy_getinfo(h, CURLINFO_RESPONSE_CODE, &resp.status)); diff --git a/src/util/curl.h b/src/util/curl.h index 7be463c0..55dfb76f 100644 --- a/src/util/curl.h +++ b/src/util/curl.h @@ -1,6 +1,8 @@ #pragma once +#include #include +#include #include #include #include @@ -57,6 +59,19 @@ struct request { std::optional body; std::optional upload_file; + // when set, the response body is streamed to this file instead of being + // buffered in memory (essential for multi-GB blob downloads). response.body + // is left empty in that case. + std::optional download_file; + + // optional download-progress callback: (bytes_downloaded, bytes_total). + // bytes_total is 0 until the server reports a content length. + std::function on_download_progress; + + // optional abort predicate, polled during transfer; returning true aborts + // the request (used to make a long download responsive to Ctrl-C). + std::function should_abort; + // follow 3xx redirects. off by default (matching libcurl), since following // is an SSRF/credential-leak/protocol-downgrade surface that only specific // operations need (e.g. a blob GET that a registry 307-redirects to cloud From 60290b6d4e2e056e77c859e00bd7cf0a9c922699 Mon Sep 17 00:00:00 2001 From: bcumming Date: Tue, 30 Jun 2026 13:29:58 +0200 Subject: [PATCH 05/29] move oci to its own path --- meson.build | 6 +++--- src/cli/pull.cpp | 4 ++-- src/{uenv => }/oci/auth.cpp | 2 -- src/{uenv => }/oci/auth.h | 2 -- src/{uenv => }/oci/client.cpp | 2 -- src/{uenv => }/oci/client.h | 4 +--- src/{uenv => }/oci/pull.cpp | 2 -- src/{uenv => }/oci/pull.h | 4 +--- test/unit/oci_auth.cpp | 14 +++++++------- test/unit/oci_client.cpp | 4 ++-- 10 files changed, 16 insertions(+), 28 deletions(-) rename src/{uenv => }/oci/auth.cpp (99%) rename src/{uenv => }/oci/auth.h (98%) rename src/{uenv => }/oci/client.cpp (99%) rename src/{uenv => }/oci/client.h (98%) rename src/{uenv => }/oci/pull.cpp (99%) rename src/{uenv => }/oci/pull.h (95%) diff --git a/meson.build b/meson.build index ceec3830..91c3bf54 100644 --- a/meson.build +++ b/meson.build @@ -62,9 +62,9 @@ lib_src = [ 'src/uenv/log.cpp', 'src/uenv/meta.cpp', 'src/uenv/mount.cpp', - 'src/uenv/oci/auth.cpp', - 'src/uenv/oci/client.cpp', - 'src/uenv/oci/pull.cpp', + 'src/oci/auth.cpp', + 'src/oci/client.cpp', + 'src/oci/pull.cpp', 'src/uenv/oras.cpp', 'src/uenv/parse.cpp', 'src/uenv/print.cpp', diff --git a/src/cli/pull.cpp b/src/cli/pull.cpp index bc8bdeec..fe717719 100644 --- a/src/cli/pull.cpp +++ b/src/cli/pull.cpp @@ -12,8 +12,8 @@ #include #include -#include -#include +#include +#include #include #include #include diff --git a/src/uenv/oci/auth.cpp b/src/oci/auth.cpp similarity index 99% rename from src/uenv/oci/auth.cpp rename to src/oci/auth.cpp index 8122f3b2..594f9434 100644 --- a/src/uenv/oci/auth.cpp +++ b/src/oci/auth.cpp @@ -13,7 +13,6 @@ #include "auth.h" -namespace uenv { namespace oci { namespace { @@ -266,4 +265,3 @@ authenticate(const std::string& registry_url, } } // namespace oci -} // namespace uenv diff --git a/src/uenv/oci/auth.h b/src/oci/auth.h similarity index 98% rename from src/uenv/oci/auth.h rename to src/oci/auth.h index a9f32c5c..e0b73585 100644 --- a/src/uenv/oci/auth.h +++ b/src/oci/auth.h @@ -7,7 +7,6 @@ #include -namespace uenv { namespace oci { // HTTP basic-auth credentials used to obtain a push/private-pull token. @@ -77,4 +76,3 @@ authenticate(const std::string& registry_url, const std::optional& creds = std::nullopt); } // namespace oci -} // namespace uenv diff --git a/src/uenv/oci/client.cpp b/src/oci/client.cpp similarity index 99% rename from src/uenv/oci/client.cpp rename to src/oci/client.cpp index d40c355d..171c3410 100644 --- a/src/uenv/oci/client.cpp +++ b/src/oci/client.cpp @@ -13,7 +13,6 @@ #include "client.h" -namespace uenv { namespace oci { // --- pure helpers ------------------------------------------------------- @@ -410,4 +409,3 @@ client::put_manifest(const std::string& reference, const std::string& body, } } // namespace oci -} // namespace uenv diff --git a/src/uenv/oci/client.h b/src/oci/client.h similarity index 98% rename from src/uenv/oci/client.h rename to src/oci/client.h index a4491d34..94096fef 100644 --- a/src/uenv/oci/client.h +++ b/src/oci/client.h @@ -9,11 +9,10 @@ #include #include -#include +#include #include #include -namespace uenv { namespace oci { // common OCI media types @@ -151,4 +150,3 @@ class client { }; } // namespace oci -} // namespace uenv diff --git a/src/uenv/oci/pull.cpp b/src/oci/pull.cpp similarity index 99% rename from src/uenv/oci/pull.cpp rename to src/oci/pull.cpp index 108b0d1b..8187eebf 100644 --- a/src/uenv/oci/pull.cpp +++ b/src/oci/pull.cpp @@ -13,7 +13,6 @@ #include "client.h" #include "pull.h" -namespace uenv { namespace oci { namespace fs = std::filesystem; @@ -192,4 +191,3 @@ pull_meta(client& c, const std::string& manifest_digest, const fs::path& store) } } // namespace oci -} // namespace uenv diff --git a/src/uenv/oci/pull.h b/src/oci/pull.h similarity index 95% rename from src/uenv/oci/pull.h rename to src/oci/pull.h index 67d1ab92..500bedf8 100644 --- a/src/uenv/oci/pull.h +++ b/src/oci/pull.h @@ -5,10 +5,9 @@ #include #include -#include +#include #include -namespace uenv { namespace oci { // progress callback for the squashfs download: (bytes_downloaded, bytes_total). @@ -32,4 +31,3 @@ pull_meta(client& c, const std::string& manifest_digest, const std::filesystem::path& store); } // namespace oci -} // namespace uenv diff --git a/test/unit/oci_auth.cpp b/test/unit/oci_auth.cpp index 3aebe10b..782c1bf6 100644 --- a/test/unit/oci_auth.cpp +++ b/test/unit/oci_auth.cpp @@ -1,11 +1,11 @@ #include -#include +#include -using uenv::oci::parse_bearer_challenge; -using uenv::oci::parse_token_response; -using uenv::oci::repository_scope; -using uenv::oci::token_url; +using oci::parse_bearer_challenge; +using oci::parse_token_response; +using oci::repository_scope; +using oci::token_url; TEST_CASE("oci parse_bearer_challenge from jfrog", "[oci][auth]") { // the exact challenge the CSCS jfrog registry returns (spike step 1) @@ -58,7 +58,7 @@ TEST_CASE("oci parse_bearer_challenge is scheme-case-insensitive", } TEST_CASE("oci token_url", "[oci][auth]") { - uenv::oci::bearer_challenge c{ + oci::bearer_challenge c{ .realm = "https://jfrog.svc.cscs.ch/v2/token", .service = "jfrog.svc.cscs.ch", .scopes = {}}; @@ -74,7 +74,7 @@ TEST_CASE("oci token_url", "[oci][auth]") { "&scope=repository:a:pull&scope=repository:b:push"); // a realm that already carries a query continues with '&' - uenv::oci::bearer_challenge q{ + oci::bearer_challenge q{ .realm = "https://r/token?foo=bar", .service = "reg", .scopes = {}}; REQUIRE(token_url(q, {}) == "https://r/token?foo=bar&service=reg"); } diff --git a/test/unit/oci_client.cpp b/test/unit/oci_client.cpp index d65bb1ef..c5eb7bbe 100644 --- a/test/unit/oci_client.cpp +++ b/test/unit/oci_client.cpp @@ -1,9 +1,9 @@ #include -#include +#include #include -using namespace uenv::oci; +using namespace oci; TEST_CASE("oci digest_string", "[oci][client]") { auto d = util::sha256_string("abc"); From 3a1d6866d12d41c0024947086339b9e11e864fb6 Mon Sep 17 00:00:00 2001 From: bcumming Date: Tue, 30 Jun 2026 14:31:46 +0200 Subject: [PATCH 06/29] clean up oras pull replacement --- src/cli/copy.cpp | 13 +++-- src/cli/delete.cpp | 9 ++-- src/cli/pull.cpp | 24 ++++----- src/cli/push.cpp | 15 ++++-- src/oci/auth.cpp | 103 +++++++++++++++++++++++------------ src/oci/auth.h | 52 +++++++++--------- src/oci/client.cpp | 40 ++++++++------ src/oci/client.h | 28 ---------- src/oci/pull.cpp | 12 +++-- src/site/site.cpp | 1 - src/uenv/oras.cpp | 51 ------------------ src/uenv/oras.h | 4 -- src/util/curl.cpp | 28 +--------- src/util/sha256.cpp | 4 +- src/util/sha256.h | 5 +- src/util/strings.cpp | 34 ++++++++---- src/util/strings.h | 13 ++++- test/unit/oci_auth.cpp | 112 +++++++++++++++++++++++++++++++++++++-- test/unit/oci_client.cpp | 16 ++++++ 19 files changed, 332 insertions(+), 232 deletions(-) diff --git a/src/cli/copy.cpp b/src/cli/copy.cpp index ab0b0647..56023179 100644 --- a/src/cli/copy.cpp +++ b/src/cli/copy.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -59,13 +60,19 @@ int image_copy([[maybe_unused]] const image_copy_args& args, } const auto& registry_cfg = *settings.config.registry; - std::optional credentials; - if (auto c = oras::get_credentials(args.username, args.token)) { + std::optional credentials; + if (auto c = oci::get_credentials(args.username, args.token)) { credentials = *c; } else { term::error("{}", c.error()); return 1; } + // copy still runs on oras; adapt the credentials at the legacy boundary. + std::optional oras_creds; + if (credentials) { + oras_creds = uenv::oras::credentials{credentials->username, + credentials->password}; + } uenv_nslabel src_label{}; if (const auto parse = parse_uenv_nslabel(args.src_uenv_description)) { @@ -175,7 +182,7 @@ int image_copy([[maybe_unused]] const image_copy_args& args, spdlog::debug("registry url: {}", rego_url); if (auto result = oras::copy(rego_url, src_label.nspace.value(), src_record, - dst_label.nspace.value(), dst_record, credentials); + dst_label.nspace.value(), dst_record, oras_creds); !result) { term::error("unable to copy uenv.\n{}", result.error().message); return 1; diff --git a/src/cli/delete.cpp b/src/cli/delete.cpp index 151fa388..40e45718 100644 --- a/src/cli/delete.cpp +++ b/src/cli/delete.cpp @@ -9,7 +9,7 @@ #include #include -#include +#include #include #include #include @@ -62,8 +62,8 @@ int image_delete([[maybe_unused]] const image_delete_args& args, } const auto& artifactory_url = *registry_cfg.artifactory_url; - uenv::oras::credentials credentials; - if (auto c = oras::get_credentials(args.username, args.token)) { + oci::credentials credentials; + if (auto c = oci::get_credentials(args.username, args.token)) { if (!*c) { term::error("full credentials must be provided", c.error()); } @@ -126,7 +126,8 @@ int image_delete([[maybe_unused]] const image_delete_args& args, record.version, record.tag); if (auto result = - util::curl::del(url, credentials.username, credentials.token); + util::curl::del(url, credentials.username, + credentials.password); !result) { term::error("unable to delete uenv: {}", result.error().message); return 1; diff --git a/src/cli/pull.cpp b/src/cli/pull.cpp index fe717719..a24dab79 100644 --- a/src/cli/pull.cpp +++ b/src/cli/pull.cpp @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -75,8 +74,8 @@ int image_pull(const image_pull_args& args, const global_settings& settings) { } const auto& registry_cfg = *settings.config.registry; - std::optional credentials; - if (auto c = oras::get_credentials(args.username, args.token)) { + std::optional credentials; + if (auto c = oci::get_credentials(args.username, args.token)) { credentials = *c; } else { term::error("{}", c.error()); @@ -202,16 +201,11 @@ int image_pull(const image_pull_args& args, const global_settings& settings) { record.system, record.uarch, record.name, record.version); - std::optional oci_creds; - if (credentials) { - oci_creds = - oci::credentials{credentials->username, credentials->token}; - } - spdlog::debug("oci pull: registry={} repository={}", registry_base, repository); - auto client = oci::client::create(registry_base, repository, oci_creds); + auto client = + oci::client::create(registry_base, repository, credentials); if (!client) { term::error("unable to connect to the registry:\n{}", client.error()); @@ -269,7 +263,6 @@ int image_pull(const image_pull_args& args, const global_settings& settings) { .no_tty = !isatty(fileno(stdout)), }); - util::set_signal_catcher(); auto progress = [&downloaded_mb](std::uint64_t now, std::uint64_t) { downloaded_mb = now / (1024 * 1024); @@ -280,12 +273,17 @@ int image_pull(const image_pull_args& args, const global_settings& settings) { // than calling signal_raised() again (which would see false and // skip the cleanup, leaving a partial download behind). bool aborted = false; + util::set_signal_catcher(); auto result = oci::pull_squashfs( - *client, *manifest, paths.store, progress, [&aborted]() { + *client, *manifest, paths.store, progress, + [&aborted]() { aborted = aborted || util::signal_raised(); return aborted; }); - downloaded_mb = total_mb; + if (!aborted) { + // ensure that the progress bar shows %100 on completion + downloaded_mb = total_mb; + } bar->done(); if (!result) { diff --git a/src/cli/push.cpp b/src/cli/push.cpp index 7e824993..3442a3f1 100644 --- a/src/cli/push.cpp +++ b/src/cli/push.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -64,13 +65,19 @@ int image_push([[maybe_unused]] const image_push_args& args, } const auto& registry_cfg = *settings.config.registry; - std::optional credentials; - if (auto c = oras::get_credentials(args.username, args.token)) { + std::optional credentials; + if (auto c = oci::get_credentials(args.username, args.token)) { credentials = *c; } else { term::error("{}", c.error()); return 1; } + // push still runs on oras; adapt the credentials at the legacy boundary. + std::optional oras_creds; + if (credentials) { + oras_creds = uenv::oras::credentials{credentials->username, + credentials->password}; + } // parse and validate the destination (i.e. the label in the registry) uenv_nslabel dst_label{}; @@ -130,7 +137,7 @@ int image_push([[maybe_unused]] const image_push_args& args, // Push the SquashFS image auto push_result = oras::push_tag(rego_url, nspace, dst_label.label, - sqfs->sqfs, credentials); + sqfs->sqfs, oras_creds); if (!push_result) { term::error("unable to push uenv.\n{}", push_result.error().message); @@ -144,7 +151,7 @@ int image_push([[maybe_unused]] const image_push_args& args, auto meta_result = oras::push_meta(rego_url, nspace, dst_label.label, - sqfs->meta.value(), credentials); + sqfs->meta.value(), oras_creds); if (!meta_result) { spdlog::warn("unable to push metadata.\n{}", meta_result.error().message); diff --git a/src/oci/auth.cpp b/src/oci/auth.cpp index 594f9434..48f69059 100644 --- a/src/oci/auth.cpp +++ b/src/oci/auth.cpp @@ -1,4 +1,7 @@ +#include + #include +#include #include #include #include @@ -10,36 +13,14 @@ #include #include - -#include "auth.h" +#include +#include +#include namespace oci { namespace { -std::string to_lower(std::string_view s) { - std::string out; - out.reserve(s.size()); - for (char c : s) { - out.push_back( - static_cast(std::tolower(static_cast(c)))); - } - return out; -} - -std::string_view trim(std::string_view s) { - auto is_space = [](char c) { - return std::isspace(static_cast(c)) != 0; - }; - while (!s.empty() && is_space(s.front())) { - s.remove_prefix(1); - } - while (!s.empty() && is_space(s.back())) { - s.remove_suffix(1); - } - return s; -} - // split a scope value on spaces (the OCI spec permits a single scope parameter // carrying a space-separated list). void append_scopes(std::vector& out, std::string_view value) { @@ -62,9 +43,14 @@ void append_scopes(std::vector& out, std::string_view value) { } // namespace +// --- pure helpers (no network; unit-tested) ----------------------------- +// Kept out of the public auth.h interface; tests redeclare these prototypes +// inside namespace oci::impl (see test/unit/oci_auth.cpp). +namespace impl { + std::optional parse_bearer_challenge(std::string_view v) { - std::string_view s = trim(v); + std::string_view s = util::trim(v); // case-insensitive "Bearer" scheme, followed by whitespace constexpr std::string_view scheme = "bearer"; @@ -80,7 +66,7 @@ parse_bearer_challenge(std::string_view v) { if (!s.empty() && !std::isspace(static_cast(s.front()))) { return std::nullopt; // e.g. "BearerX" is not the Bearer scheme } - s = trim(s); + s = util::trim(s); bearer_challenge challenge; @@ -91,7 +77,8 @@ parse_bearer_challenge(std::string_view v) { while (i < s.size() && s[i] != '=' && s[i] != ',') { ++i; } - std::string key = to_lower(trim(s.substr(key_start, i - key_start))); + std::string key = + util::to_lower(util::trim(s.substr(key_start, i - key_start))); std::string value; if (i < s.size() && s[i] == '=') { @@ -117,7 +104,8 @@ parse_bearer_challenge(std::string_view v) { while (i < s.size() && s[i] != ',') { ++i; } - value = std::string(trim(s.substr(val_start, i - val_start))); + value = std::string( + util::trim(s.substr(val_start, i - val_start))); } } @@ -136,7 +124,7 @@ parse_bearer_challenge(std::string_view v) { if (i < s.size() && s[i] == ',') { ++i; } - s = trim(s.substr(i)); + s = util::trim(s.substr(i)); i = 0; } @@ -184,6 +172,8 @@ std::string repository_scope(std::string_view repository, return fmt::format("repository:{}:{}", repository, actions); } +} // namespace impl + util::expected discover_challenge(const std::string& registry_url) { // normalise: drop any trailing '/' before appending the v2 base path. @@ -215,7 +205,7 @@ discover_challenge(const std::string& registry_url) { return util::unexpected{fmt::format( "{} returned 401 with no WWW-Authenticate header", req.url)}; } - auto challenge = parse_bearer_challenge(*header); + auto challenge = impl::parse_bearer_challenge(*header); if (!challenge) { return util::unexpected{fmt::format( "could not parse WWW-Authenticate header: {}", *header)}; @@ -228,7 +218,7 @@ fetch_token(const bearer_challenge& challenge, const std::vector& scopes, const std::optional& creds) { util::curl::request req; - req.url = token_url(challenge, scopes); + req.url = impl::token_url(challenge, scopes); if (creds) { req.username = creds->username; req.password = creds->password; @@ -245,7 +235,7 @@ fetch_token(const bearer_challenge& challenge, fmt::format("token request to {} failed with status {}: {}", req.url, resp->status, resp->body)}; } - auto token = parse_token_response(resp->body); + auto token = impl::parse_token_response(resp->body); if (!token) { return util::unexpected{ "token endpoint response did not contain a token"}; @@ -264,4 +254,51 @@ authenticate(const std::string& registry_url, return fetch_token(*challenge, scopes, creds); } +util::expected, std::string> +get_credentials(std::optional username, + std::optional token) { + namespace fs = std::filesystem; + + if (!token) { + return std::nullopt; + } + + fs::path token_path{token.value()}; + if (!fs::exists(token_path)) { + return util::unexpected{fmt::format( + "the token '{}' is not a path or file.", token_path.string())}; + } + + if (fs::is_directory(token_path)) { + token_path = token_path / "TOKEN"; + if (!fs::exists(token_path)) { + return util::unexpected{fmt::format( + "the token file '{}' does not exist.", token_path.string())}; + } + } + + if (util::file_access_level(token_path) < util::file_level::readonly) { + return util::unexpected{fmt::format( + "you do not have permission to read the token file '{}'", + token_path.string())}; + } + + auto token_string = util::read_single_line_file(token_path); + if (!token_string) { + return util::unexpected{fmt::format("unable to read a token from '{}'", + token_path.string())}; + } + + if (username) { + return credentials{.username = username.value(), + .password = *token_string}; + } + if (auto name = getlogin()) { + return credentials{.username = name, .password = *token_string}; + } + + return util::unexpected{ + "provide a username with --username for the --token."}; +} + } // namespace oci diff --git a/src/oci/auth.h b/src/oci/auth.h index e0b73585..b2bf3abc 100644 --- a/src/oci/auth.h +++ b/src/oci/auth.h @@ -29,31 +29,6 @@ struct bearer_challenge { const bearer_challenge&) = default; }; -// --- pure helpers (no network; unit-tested) ----------------------------- - -// Parse the value of a `WWW-Authenticate` header. Returns nullopt if it is not -// a Bearer challenge or has no realm. Handles quoted values that themselves -// contain commas (e.g. scope="pull,push") and space-separated scope lists. -std::optional -parse_bearer_challenge(std::string_view header_value); - -// Build the token-request URL: ?service=&scope=&scope=. -// The scopes we intend to use are passed explicitly (the challenge's own scope -// is only advisory). OCI repository names and pull/push actions contain only -// query-safe characters, so values are placed verbatim. -std::string token_url(const bearer_challenge& challenge, - const std::vector& scopes); - -// Extract the bearer token from a token-endpoint JSON body, accepting both the -// `{"token":...}` and `{"access_token":...}` spellings. nullopt if neither is -// present or the body is not valid JSON. -std::optional parse_token_response(std::string_view body); - -// Compose an OCI auth scope string: repository:: -// e.g. repository_scope("deploy/todi/gh200/app/1.0", "pull,push"). -std::string repository_scope(std::string_view repository, - std::string_view actions); - // --- network operations ------------------------------------------------- // Probe `/v2/` and parse the Bearer challenge from the 401. @@ -75,4 +50,31 @@ authenticate(const std::string& registry_url, const std::vector& scopes, const std::optional& creds = std::nullopt); +// --- credential resolution ---------------------------------------------- + +// Resolve registry credentials from CLI arguments. `token` is a path to a file +// holding the token string (its first line is read); if `token` names a +// directory, its `TOKEN` entry is used. The username is taken from `username` +// when set, otherwise from the OS login name. Returns `std::nullopt` when no +// `token` is given (anonymous access), or an error string describing why the +// token could not be read. +util::expected, std::string> +get_credentials(std::optional username, + std::optional token); + } // namespace oci + +#include +// Formats credentials with the password redacted, for safe logging. +template <> class fmt::formatter { + public: + constexpr auto parse(format_parse_context& ctx) { + return ctx.end(); + } + template + constexpr auto format(oci::credentials const& c, FmtContext& ctx) const { + // replace password characters with 'X' + return fmt::format_to(ctx.out(), "{{username: {}, password: {:X>{}}}}", + c.username, "", c.password.size()); + } +}; diff --git a/src/oci/client.cpp b/src/oci/client.cpp index 171c3410..b3cb6d1c 100644 --- a/src/oci/client.cpp +++ b/src/oci/client.cpp @@ -10,12 +10,18 @@ #include #include - -#include "client.h" +#include namespace oci { -// --- pure helpers ------------------------------------------------------- +// pure helpers (no network; unit-tested). Kept out of the public client.h +// interface; tests redeclare these prototypes inside namespace oci::impl (see +// test/unit/oci_client.cpp). +namespace impl { + +// defined in auth.cpp (also part of oci::impl). +std::string repository_scope(std::string_view repository, + std::string_view actions); std::string digest_string(const util::sha256_digest& d) { return "sha256:" + d.hex(); @@ -112,6 +118,8 @@ std::optional> parse_referrers(std::string_view body) { return out; } +} // namespace impl + // --- client ------------------------------------------------------------- util::expected @@ -138,7 +146,7 @@ util::expected client::token_for(bool write) { if (cache) { return *cache; } - auto scope = repository_scope(repository_, write ? "pull,push" : "pull"); + auto scope = impl::repository_scope(repository_, write ? "pull,push" : "pull"); auto token = fetch_token(challenge_, {scope}, creds_); if (!token) { return util::unexpected{token.error()}; @@ -154,7 +162,7 @@ client::blob_exists(const std::string& digest) { return util::unexpected{token.error()}; } util::curl::request req; - req.url = registry_url_ + blob_path(repository_, digest); + req.url = registry_url_ + impl::blob_path(repository_, digest); req.method = util::curl::http_method::head; req.bearer_token = *token; @@ -179,7 +187,7 @@ client::get_blob(const std::string& digest) { return util::unexpected{token.error()}; } util::curl::request req; - req.url = registry_url_ + blob_path(repository_, digest); + req.url = registry_url_ + impl::blob_path(repository_, digest); req.bearer_token = *token; // registries 307-redirect blob downloads to backing storage. req.follow_redirects = true; @@ -205,7 +213,7 @@ client::get_blob_to_file( return util::unexpected{token.error()}; } util::curl::request req; - req.url = registry_url_ + blob_path(repository_, digest); + req.url = registry_url_ + impl::blob_path(repository_, digest); req.bearer_token = *token; req.follow_redirects = true; req.download_file = file; @@ -230,7 +238,7 @@ client::get_manifest(const std::string& reference) { return util::unexpected{token.error()}; } util::curl::request req; - req.url = registry_url_ + manifest_path(repository_, reference); + req.url = registry_url_ + impl::manifest_path(repository_, reference); req.bearer_token = *token; req.header_lines = {fmt::format("Accept: {}", media_type_manifest), fmt::format("Accept: {}", media_type_index)}; @@ -257,7 +265,7 @@ util::expected, std::string> client::list_tags() { return util::unexpected{token.error()}; } util::curl::request req; - req.url = registry_url_ + tags_path(repository_); + req.url = registry_url_ + impl::tags_path(repository_); req.bearer_token = *token; auto resp = util::curl::perform(req); @@ -268,7 +276,7 @@ util::expected, std::string> client::list_tags() { return util::unexpected{fmt::format( "failed to list tags (status {})", resp->status)}; } - auto tags = parse_tags_list(resp->body); + auto tags = impl::parse_tags_list(resp->body); if (!tags) { return util::unexpected{"could not parse tags/list response"}; } @@ -282,7 +290,7 @@ client::referrers(const std::string& digest) { return util::unexpected{token.error()}; } util::curl::request req; - req.url = registry_url_ + referrers_path(repository_, digest); + req.url = registry_url_ + impl::referrers_path(repository_, digest); req.bearer_token = *token; auto resp = util::curl::perform(req); @@ -294,7 +302,7 @@ client::referrers(const std::string& digest) { "failed to fetch referrers of {} (status {})", digest, resp->status)}; } - auto refs = parse_referrers(resp->body); + auto refs = impl::parse_referrers(resp->body); if (!refs) { return util::unexpected{"could not parse referrers response"}; } @@ -331,7 +339,7 @@ do_put_blob(const std::string& registry_url, const std::string& uploads, // 2. PUT the payload to ?digest= util::curl::request put; - put.url = resolve_upload_url(registry_url, *location, digest); + put.url = impl::resolve_upload_url(registry_url, *location, digest); put.method = util::curl::http_method::put; put.bearer_token = token; put.header_lines = { @@ -363,7 +371,7 @@ client::put_blob(const std::string& digest, if (!token) { return util::unexpected{token.error()}; } - return do_put_blob(registry_url_, uploads_path(repository_), *token, digest, + return do_put_blob(registry_url_, impl::uploads_path(repository_), *token, digest, [&file](util::curl::request& r) { r.upload_file = file; }); @@ -378,7 +386,7 @@ client::put_blob_bytes(const std::string& digest, const std::string& data) { if (!token) { return util::unexpected{token.error()}; } - return do_put_blob(registry_url_, uploads_path(repository_), *token, digest, + return do_put_blob(registry_url_, impl::uploads_path(repository_), *token, digest, [&data](util::curl::request& r) { r.body = data; }); } @@ -390,7 +398,7 @@ client::put_manifest(const std::string& reference, const std::string& body, return util::unexpected{token.error()}; } util::curl::request req; - req.url = registry_url_ + manifest_path(repository_, reference); + req.url = registry_url_ + impl::manifest_path(repository_, reference); req.method = util::curl::http_method::put; req.bearer_token = *token; req.body = body; diff --git a/src/oci/client.h b/src/oci/client.h index 94096fef..f1510191 100644 --- a/src/oci/client.h +++ b/src/oci/client.h @@ -42,34 +42,6 @@ struct manifest_response { std::string media_type; }; -// --- pure helpers (no network; unit-tested) ----------------------------- - -// "sha256:" for a digest. -std::string digest_string(const util::sha256_digest& d); - -// OCI Distribution v2 path builders (each begins with "/v2/"). -std::string blob_path(std::string_view repository, std::string_view digest); -std::string manifest_path(std::string_view repository, - std::string_view reference); -std::string uploads_path(std::string_view repository); -std::string tags_path(std::string_view repository); -std::string referrers_path(std::string_view repository, - std::string_view digest); - -// Resolve a blob-upload Location (which may be absolute or registry-relative) -// against the registry base URL, then append the monolithic-upload digest query -// parameter, choosing '?' or '&' as appropriate. -std::string resolve_upload_url(std::string_view registry_url, - std::string_view location, - std::string_view digest); - -// Parse a /tags/list body ({"name":...,"tags":[...]}) into the tag list. -std::optional> -parse_tags_list(std::string_view body); - -// Parse a referrers index body into its manifest descriptors. -std::optional> parse_referrers(std::string_view body); - // --- registry client ---------------------------------------------------- // A client bound to one repository on one registry. Authenticates lazily, diff --git a/src/oci/pull.cpp b/src/oci/pull.cpp index 8187eebf..af96406e 100644 --- a/src/oci/pull.cpp +++ b/src/oci/pull.cpp @@ -9,12 +9,16 @@ #include #include #include - -#include "client.h" -#include "pull.h" +#include +#include namespace oci { +// defined in client.cpp (part of oci::impl, kept out of the public interface). +namespace impl { +std::string digest_string(const util::sha256_digest& d); +} + namespace fs = std::filesystem; namespace { @@ -111,7 +115,7 @@ pull_squashfs(client& c, const manifest_response& manifest, if (!actual) { return util::unexpected{actual.error()}; } - auto actual_digest = digest_string(*actual); + auto actual_digest = impl::digest_string(*actual); if (actual_digest != digest) { std::error_code ec; fs::remove(dest, ec); diff --git a/src/site/site.cpp b/src/site/site.cpp index dc07068c..b65bde8c 100644 --- a/src/site/site.cpp +++ b/src/site/site.cpp @@ -3,7 +3,6 @@ #include #include -#include #include #include #include diff --git a/src/uenv/oras.cpp b/src/uenv/oras.cpp index 006169a4..4063f136 100644 --- a/src/uenv/oras.cpp +++ b/src/uenv/oras.cpp @@ -1,7 +1,6 @@ #include #include -#include #include #include @@ -501,55 +500,5 @@ copy(const std::string& registry, const std::string& src_nspace, return {}; } -util::expected, std::string> -get_credentials(std::optional username, - std::optional token) { - namespace fs = std::filesystem; - namespace oras = uenv::oras; - - if (!token) { - return std::nullopt; - } - - fs::path token_path{token.value()}; - if (!fs::exists(token_path)) { - return util::unexpected{fmt::format( - "the token '{}' is not a path or file.", token_path.string())}; - } - - if (fs::is_directory(token_path)) { - token_path = token_path / "TOKEN"; - if (!fs::exists(token_path)) { - return util::unexpected{fmt::format( - "the token file '{}' does not exist.", token_path.string())}; - } - } - - if (util::file_access_level(token_path) < util::file_level::readonly) { - return util::unexpected{fmt::format( - "you do not have permission to read the token file '{}'", - token_path.string())}; - } - - std::string token_string{}; - if (std::ifstream fid{token_path}) { - std::getline(fid, token_string); - } else { - return util::unexpected{fmt::format("unable to read a token from '{}'", - token_path.string())}; - } - - if (username) { - return oras::credentials{.username = username.value(), - .token = token_string}; - } - if (auto name = getlogin()) { - return oras::credentials{.username = name, .token = token_string}; - } - - return util::unexpected{ - "provide a username with --username for the --token."}; -} - } // namespace oras } // namespace uenv diff --git a/src/uenv/oras.h b/src/uenv/oras.h index fc1c7fdb..2eb5d717 100644 --- a/src/uenv/oras.h +++ b/src/uenv/oras.h @@ -66,10 +66,6 @@ copy(const std::string& registry, const std::string& src_nspace, const uenv_record& dst_uenv, const std::optional token = std::nullopt); -util::expected, std::string> -get_credentials(std::optional username, - std::optional token); - } // namespace oras } // namespace uenv diff --git a/src/util/curl.cpp b/src/util/curl.cpp index f45fd675..8b8164f3 100644 --- a/src/util/curl.cpp +++ b/src/util/curl.cpp @@ -12,38 +12,12 @@ #include #include #include +#include namespace util { namespace curl { -namespace { - -std::string to_lower(std::string_view s) { - std::string out; - out.reserve(s.size()); - for (char c : s) { - out.push_back(static_cast( - std::tolower(static_cast(c)))); - } - return out; -} - -std::string_view trim(std::string_view s) { - auto is_space = [](char c) { - return std::isspace(static_cast(c)) != 0; - }; - while (!s.empty() && is_space(s.front())) { - s.remove_prefix(1); - } - while (!s.empty() && is_space(s.back())) { - s.remove_suffix(1); - } - return s; -} - -} // namespace - std::optional headers::get(std::string_view name) const { if (auto it = entries.find(to_lower(name)); it != entries.end()) { return it->second; diff --git a/src/util/sha256.cpp b/src/util/sha256.cpp index 0d71dc64..8cb87f90 100644 --- a/src/util/sha256.cpp +++ b/src/util/sha256.cpp @@ -31,8 +31,8 @@ inline uint32_t rotr(uint32_t x, uint32_t n) { return (x >> n) | (x << (32 - n)); } -// The real digest state. Lives only in this translation unit; callers see it as -// the opaque sha256_state blob. +// The digest state. +// Callers see it as the opaque sha256_state blob. struct sha256_impl { uint32_t state[8]; uint64_t bitlen; diff --git a/src/util/sha256.h b/src/util/sha256.h index 4adbb992..80da53f0 100644 --- a/src/util/sha256.h +++ b/src/util/sha256.h @@ -27,10 +27,11 @@ struct sha256_digest { // // An opaque state blob you sha256_init() once, feed bytes into with // sha256_update(), then sha256_final(). Stack-allocatable and trivially -// copyable; the internal layout lives entirely in sha256.cpp, so the FIPS -// implementation never leaks into this header. Use this when data must be +// copyable. +// Use this when data must be // digested incrementally (e.g. streaming a multi-GB squashfs) without holding // it all in memory. +// implementation never leaks into this header. struct sha256_state { // opaque storage - do not inspect or modify directly. sized and aligned to // hold the implementation state (asserted in sha256.cpp). diff --git a/src/util/strings.cpp b/src/util/strings.cpp index d1ce9794..ea5aab2a 100644 --- a/src/util/strings.cpp +++ b/src/util/strings.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -8,18 +9,31 @@ namespace util { -std::string strip(std::string_view input) { - if (input.empty()) { - return {}; +std::string_view trim(std::string_view s) { + auto is_space = [](char c) { + return std::isspace(static_cast(c)) != 0; + }; + while (!s.empty() && is_space(s.front())) { + s.remove_prefix(1); + } + while (!s.empty() && is_space(s.back())) { + s.remove_suffix(1); } - auto b = std::find_if_not(input.begin(), input.end(), std::iswspace); - if (b == input.end()) { - return {}; + return s; +} + +std::string strip(std::string_view input) { + return std::string{trim(input)}; +} + +std::string to_lower(std::string_view s) { + std::string out; + out.reserve(s.size()); + for (char c : s) { + out.push_back( + static_cast(std::tolower(static_cast(c)))); } - auto e = input.end(); - while (std::iswspace(*--e)) - ; - return {b, e + 1}; + return out; } std::vector split(std::string_view s, const char delim, diff --git a/src/util/strings.h b/src/util/strings.h index 4dbec1ce..3871a78d 100644 --- a/src/util/strings.h +++ b/src/util/strings.h @@ -1,4 +1,7 @@ +#pragma once + #include +#include #include namespace util { @@ -31,9 +34,17 @@ namespace util { std::vector split(std::string_view s, const char delim, const bool drop_empty = false); -// strip whitespace from beginning and end of a string +// remove leading and trailing whitespace, returning a view into `input` (no +// allocation). The returned view is only valid while `input` lives. +std::string_view trim(std::string_view input); + +// strip whitespace from beginning and end of a string (the owning-string form +// of trim()). std::string strip(std::string_view input); +// lower-case each (ASCII) character. +std::string to_lower(std::string_view input); + std::string join(std::string_view joiner, const std::vector& list); bool is_sha(const std::string& str); diff --git a/test/unit/oci_auth.cpp b/test/unit/oci_auth.cpp index 782c1bf6..67b76343 100644 --- a/test/unit/oci_auth.cpp +++ b/test/unit/oci_auth.cpp @@ -1,11 +1,25 @@ +#include +#include + #include +#include #include +#include + +// the pure helpers are not part of the public auth.h interface; redeclare their +// prototypes here (they live in namespace oci::impl in auth.cpp). +namespace oci::impl { +std::optional parse_bearer_challenge(std::string_view); +std::string token_url(const bearer_challenge&, const std::vector&); +std::optional parse_token_response(std::string_view); +std::string repository_scope(std::string_view, std::string_view); +} // namespace oci::impl -using oci::parse_bearer_challenge; -using oci::parse_token_response; -using oci::repository_scope; -using oci::token_url; +using oci::impl::parse_bearer_challenge; +using oci::impl::parse_token_response; +using oci::impl::repository_scope; +using oci::impl::token_url; TEST_CASE("oci parse_bearer_challenge from jfrog", "[oci][auth]") { // the exact challenge the CSCS jfrog registry returns (spike step 1) @@ -95,3 +109,93 @@ TEST_CASE("oci repository_scope", "[oci][auth]") { "repository:deploy/todi/gh200/app/1.0:pull"); REQUIRE(repository_scope("foo", "pull,push") == "repository:foo:pull,push"); } + +namespace { +// write `content` to `path`, creating/truncating it. +void write_file(const std::filesystem::path& path, std::string_view content) { + std::ofstream f{path}; + f << content; +} +} // namespace + +TEST_CASE("oci get_credentials reads a token file", "[oci][auth]") { + auto dir = util::make_temp_dir(); + auto token_file = dir / "mytoken"; + // only the first line is used as the token + write_file(token_file, "s3cr3t-token\nsecond line ignored\n"); + + auto c = oci::get_credentials(std::optional{"alice"}, + std::optional{token_file.string()}); + REQUIRE(c); // no error + REQUIRE(c->has_value()); // credentials resolved + REQUIRE((*c)->username == "alice"); + REQUIRE((*c)->password == "s3cr3t-token"); +} + +TEST_CASE("oci get_credentials with no token is anonymous", "[oci][auth]") { + // no --token -> nullopt credentials (anonymous), never an error, even when a + // username is supplied. + auto anon = oci::get_credentials(std::nullopt, std::nullopt); + REQUIRE(anon); + REQUIRE_FALSE(anon->has_value()); + + auto anon_user = + oci::get_credentials(std::optional{"bob"}, std::nullopt); + REQUIRE(anon_user); + REQUIRE_FALSE(anon_user->has_value()); +} + +TEST_CASE("oci get_credentials errors on a missing token path", + "[oci][auth]") { + auto c = + oci::get_credentials(std::optional{"alice"}, + std::optional{"/no/such/path-xyz123"}); + REQUIRE_FALSE(c); // a --token that is not a path/file is an error +} + +TEST_CASE("oci get_credentials reads /TOKEN", "[oci][auth]") { + auto dir = util::make_temp_dir(); + write_file(dir / "TOKEN", "dir-token\n"); + + // passing a directory reads its TOKEN entry + auto c = oci::get_credentials(std::optional{"alice"}, + std::optional{dir.string()}); + REQUIRE(c); + REQUIRE(c->has_value()); + REQUIRE((*c)->password == "dir-token"); +} + +TEST_CASE("oci get_credentials errors when /TOKEN is absent", + "[oci][auth]") { + auto dir = util::make_temp_dir(); // empty directory, no TOKEN + auto c = oci::get_credentials(std::optional{"alice"}, + std::optional{dir.string()}); + REQUIRE_FALSE(c); +} + +TEST_CASE("oci get_credentials falls back to the login name", "[oci][auth]") { + auto dir = util::make_temp_dir(); + auto token_file = dir / "tok"; + write_file(token_file, "tok\n"); + + auto c = oci::get_credentials(std::nullopt, + std::optional{token_file.string()}); + // getlogin() resolves a username in a normal session, but may be empty in a + // detached environment (e.g. some CI) -> the function then reports an error. + // Assert deterministically on both outcomes. + if (c) { + REQUIRE(c->has_value()); + REQUIRE_FALSE((*c)->username.empty()); + REQUIRE((*c)->password == "tok"); + } else { + REQUIRE(c.error().find("username") != std::string::npos); + } +} + +TEST_CASE("oci credentials formatter redacts the password", "[oci][auth]") { + oci::credentials c{.username = "alice", .password = "secret"}; + auto s = fmt::format("{}", c); + REQUIRE(s.find("alice") != std::string::npos); + REQUIRE(s.find("secret") == std::string::npos); // password must not leak + REQUIRE(s.find("XXXXXX") != std::string::npos); // 6 chars -> 6 'X' +} diff --git a/test/unit/oci_client.cpp b/test/unit/oci_client.cpp index c5eb7bbe..6a12aed8 100644 --- a/test/unit/oci_client.cpp +++ b/test/unit/oci_client.cpp @@ -3,7 +3,23 @@ #include #include +// the pure helpers are not part of the public client.h interface; redeclare +// their prototypes here (they live in namespace oci::impl in client.cpp). +namespace oci::impl { +std::string digest_string(const util::sha256_digest&); +std::string blob_path(std::string_view, std::string_view); +std::string manifest_path(std::string_view, std::string_view); +std::string uploads_path(std::string_view); +std::string tags_path(std::string_view); +std::string referrers_path(std::string_view, std::string_view); +std::string resolve_upload_url(std::string_view, std::string_view, + std::string_view); +std::optional> parse_tags_list(std::string_view); +std::optional> parse_referrers(std::string_view); +} // namespace oci::impl + using namespace oci; +using namespace oci::impl; TEST_CASE("oci digest_string", "[oci][client]") { auto d = util::sha256_string("abc"); From 08ef9ff060710fb9ca761994f5bd1bf9482d105d Mon Sep 17 00:00:00 2001 From: bcumming Date: Wed, 1 Jul 2026 12:02:42 +0200 Subject: [PATCH 07/29] remove stale oras wrappers --- src/uenv/oras.cpp | 142 ---------------------------------------------- src/uenv/oras.h | 18 ------ 2 files changed, 160 deletions(-) diff --git a/src/uenv/oras.cpp b/src/uenv/oras.cpp index 4063f136..41ad0879 100644 --- a/src/uenv/oras.cpp +++ b/src/uenv/oras.cpp @@ -7,7 +7,6 @@ #include #include #include -#include #include #include @@ -155,147 +154,6 @@ run_oras_async(std::vector args, return util::run(args, runpath); } -util::expected, error> -discover(const std::string& registry, const std::string& nspace, - const uenv_record& uenv, const opt_creds token) { - auto address = - fmt::format("{}/{}/{}/{}/{}/{}:{}", registry, nspace, uenv.system, - uenv.uarch, uenv.name, uenv.version, uenv.tag); - - std::vector args = {"discover", "--format", "json", - "--artifact-type", "uenv/meta", address}; - if (token) { - args.push_back("--password"); - args.push_back(token->token); - args.push_back("--username"); - args.push_back(token->username); - } - auto result = run_oras(args); - - if (result.returncode) { - spdlog::error("oras discover returncode={} stderr='{}'", - result.returncode, result.stderr); - return util::unexpected{create_error(result)}; - } - - std::vector manifests; - using json = nlohmann::json; - try { - const auto raw = json::parse(result.stdout); - for (const auto& j : raw["manifests"]) { - manifests.push_back(j["digest"]); - } - } catch (std::exception& e) { - spdlog::error("unable to parse oras discover json: {}", e.what()); - return util::unexpected(generic_error(e.what())); - } - - return manifests; -} - -util::expected -pull_digest(const std::string& registry, const std::string& nspace, - const uenv_record& uenv, const std::string& digest, - const std::filesystem::path& destination, const opt_creds token) { - auto address = - fmt::format("{}/{}/{}/{}/{}/{}@{}", registry, nspace, uenv.system, - uenv.uarch, uenv.name, uenv.version, digest); - - spdlog::debug("oras::pull_digest: {}", address); - - std::vector args{"pull", "--output", destination.string(), - address}; - if (token) { - args.push_back("--password"); - args.push_back(token->token); - args.push_back("--username"); - args.push_back(token->username); - } - auto proc = run_oras(args); - - if (proc.returncode) { - spdlog::error("unable to pull digest with oras: {}", proc.stderr); - return util::unexpected{create_error(proc)}; - } - - return {}; -} - -util::expected pull_tag(const std::string& registry, - const std::string& nspace, - const uenv_record& uenv, - const std::filesystem::path& destination, - const opt_creds token) { - using namespace std::chrono_literals; - namespace fs = std::filesystem; - namespace bk = barkeep; - - auto address = - fmt::format("{}/{}/{}/{}/{}/{}:{}", registry, nspace, uenv.system, - uenv.uarch, uenv.name, uenv.version, uenv.tag); - - spdlog::debug("oras::pull_tag: {}", address); - std::vector args{"pull", "--concurrency", "10", - "--output", destination.string(), address}; - if (token) { - args.push_back("--password"); - args.push_back(token->token); - args.push_back("--username"); - args.push_back(token->username); - } - auto proc = run_oras_async(args); - - if (!proc) { - spdlog::error("unable to pull tag with oras: {}", proc.error()); - return util::unexpected{generic_error(proc.error())}; - } - - const fs::path& sqfs = destination / "store.squashfs"; - - std::size_t downloaded_mb{0u}; - // force rounding up, so that total_mb is never zero - std::size_t total_mb{(uenv.size_byte + (1024 * 1024 - 1)) / (1024 * 1024)}; - const unsigned interval_ms = 500; - spdlog::info("byte {} MB {}", uenv.size_byte, total_mb); - auto bar = bk::ProgressBar( - &downloaded_mb, - { - .total = total_mb, - .message = fmt::format("pulling {}", uenv.id.string()), - .speed = 0.1, - .speed_unit = "MB/s", - .style = color::use_color() ? bk::ProgressBarStyle::Rich - : bk::ProgressBarStyle::Bars, - .interval = interval_ms / 1000., - .no_tty = !isatty(fileno(stdout)), - }); - - util::set_signal_catcher(); - while (!proc->finished()) { - std::this_thread::sleep_for(1ms * interval_ms); - // handle a signal, usually SIGTERM or SIGINT - if (util::signal_raised()) { - spdlog::error("signal raised - interrupting download"); - proc->kill(); - throw util::signal_exception(util::last_signal_raised()); - } - if (fs::is_regular_file(sqfs)) { - auto downloaded_bytes = fs::file_size(sqfs); - downloaded_mb = downloaded_bytes / (1024 * 1024); - } - } - downloaded_mb = total_mb; - - if (proc->rvalue()) { - spdlog::error("unable to pull tag with oras: {}", proc->err.string()); - return util::unexpected{create_error({.returncode = proc->rvalue(), - .stdout = proc->out.string(), - .stderr = proc->err.string()})}; - } - - return {}; -} - util::expected push_tag(const std::string& registry, const std::string& nspace, const uenv_label& label, diff --git a/src/uenv/oras.h b/src/uenv/oras.h index 2eb5d717..96180f38 100644 --- a/src/uenv/oras.h +++ b/src/uenv/oras.h @@ -32,24 +32,6 @@ struct error { }; }; -bool pull(std::string rego, std::string nspace); - -util::expected, error> -discover(const std::string& registry, const std::string& nspace, - const uenv_record& uenv, - const std::optional token = std::nullopt); - -util::expected -pull_digest(const std::string& registry, const std::string& nspace, - const uenv_record& uenv, const std::string& digest, - const std::filesystem::path& destination, - const std::optional token = std::nullopt); - -util::expected -pull_tag(const std::string& registry, const std::string& nspace, - const uenv_record& uenv, const std::filesystem::path& destination, - const std::optional token = std::nullopt); - util::expected push_tag(const std::string& registry, const std::string& nspace, const uenv_label& label, const std::filesystem::path& source, From 25d249c5001f8922c52992f0710a956164425808 Mon Sep 17 00:00:00 2001 From: bcumming Date: Sat, 4 Jul 2026 14:58:12 +0200 Subject: [PATCH 08/29] replace all functionality of oras push etc; clean up --- .gitignore | 3 - AGENTS.md | 1 + CLAUDE.md | 258 +++++++++++++++++++ meson.build | 6 + src/cli/copy.cpp | 34 ++- src/cli/pull.cpp | 44 ++-- src/cli/push.cpp | 107 +++++--- src/oci/auth.cpp | 119 +-------- src/oci/client.cpp | 175 +++++++++---- src/oci/client.h | 72 ++++-- src/oci/digest.cpp | 45 ++++ src/oci/digest.h | 69 +++++ src/oci/manifest.cpp | 212 ++++++++++++++++ src/oci/manifest.h | 101 ++++++++ src/oci/parse.cpp | 508 +++++++++++++++++++++++++++++++++++++ src/oci/parse.h | 63 +++++ src/oci/pull.cpp | 135 ++++------ src/oci/pull.h | 14 +- src/oci/push.cpp | 431 +++++++++++++++++++++++++++++++ src/oci/push.h | 45 ++++ src/oci/reference.cpp | 26 ++ src/oci/reference.h | 54 ++++ src/uenv/parse.cpp | 118 ++++----- src/uenv/parse.h | 38 +-- src/util/lex.cpp | 60 +++++ src/util/lex.h | 45 +++- src/util/parse.cpp | 15 ++ src/util/parse.h | 80 ++++++ subprojects/wrapdb.json | 1 + test/meson.build | 3 + test/unit/lex.cpp | 31 ++- test/unit/oci_auth.cpp | 56 +--- test/unit/oci_client.cpp | 32 ++- test/unit/oci_digest.cpp | 87 +++++++ test/unit/oci_manifest.cpp | 178 +++++++++++++ test/unit/oci_parse.cpp | 189 ++++++++++++++ 36 files changed, 2932 insertions(+), 523 deletions(-) create mode 120000 AGENTS.md create mode 100644 CLAUDE.md create mode 100644 src/oci/digest.cpp create mode 100644 src/oci/digest.h create mode 100644 src/oci/manifest.cpp create mode 100644 src/oci/manifest.h create mode 100644 src/oci/parse.cpp create mode 100644 src/oci/parse.h create mode 100644 src/oci/push.cpp create mode 100644 src/oci/push.h create mode 100644 src/oci/reference.cpp create mode 100644 src/oci/reference.h create mode 100644 src/util/parse.cpp create mode 100644 src/util/parse.h create mode 100644 subprojects/wrapdb.json create mode 100644 test/unit/oci_digest.cpp create mode 100644 test/unit/oci_manifest.cpp create mode 100644 test/unit/oci_parse.cpp diff --git a/.gitignore b/.gitignore index fbb331f6..59a4357f 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,6 @@ subprojects/sqlite-amalgamation-* subprojects/nlohmann_json-* subprojects/tomlplusplus-* subprojects/zlib-* -subprojects/wrapdb.json subprojects/.wraplock # logs and artifacts generated by packaging workflows @@ -24,5 +23,3 @@ install build pyenv - -CLAUDE.md diff --git a/AGENTS.md b/AGENTS.md new file mode 120000 index 00000000..681311eb --- /dev/null +++ b/AGENTS.md @@ -0,0 +1 @@ +CLAUDE.md \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..4947353e --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,258 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) and other coding +agents when working with code in this repository. It is the single source of +truth; `AGENTS.md` is a symlink to this file. + +## Project Overview + +uenv2 is a C++20 rewrite of uenv, a tool for managing user environments on HPC systems (specifically CSCS Alps). It provides: +- CLI tool (`uenv`) for managing and running environments from SquashFS images +- Slurm plugin for environment integration with job scheduling +- Optional `squashfs-mount` setuid helper for mounting SquashFS images + +The software is deployed as static binaries. +All environment modifications must be done via `uenv run`, `uenv start`, or Slurm integration. + +## Build System + +This project uses **Meson** (>= 1.4) as its build system with **Ninja** as the backend. + +### Building + +```bash +# Configure build (from repository root) +mkdir build && cd build +meson setup -Dtests=enabled .. + +# Compile +meson compile + +# Install (requires sudo for system installation) +sudo meson install --no-rebuild --skip-subprojects + +# Install to staging directory (for testing) +sudo meson install --destdir=$PWD/staging --no-rebuild --skip-subprojects +``` + +### Build Options + +Configure via `-Doption=value` with `meson setup`: +- `tests=enabled|disabled` - Enable test suite (default: disabled) +- `cli=true|false` - Build CLI tool (default: true) +- `slurm_plugin=true|false` - Build Slurm plugin (default: true) +- `squashfs_mount=true|false` - Build squashfs-mount helper (default: false) +- `oras_version=X.Y.Z` - ORAS version to download (default: 1.2.0) + +## Testing + +Four test suites exist: +1. **unit** - C++ unit tests using Catch2 (in `test/unit/`) +2. **cli** - CLI integration tests using BATS (in `test/integration/cli.bats`) +3. **slurm** - Slurm plugin tests using BATS (in `test/integration/slurm.bats`) +4. **squashfs-mount** - setuid helper tests using BATS (in `test/integration/squashfs-mount.bats`) + +### Running Tests + +To run the tests, run the tests directly instead of running them through meson. + +```bash +# Run tests directly +./test/unit # Unit tests +./test/bats ./test/cli.bats # CLI tests +./test/bats ./test/slurm.bats # Slurm tests +``` + +### Elastic mock (`elastic_mock`) + +The slurm tests include an elastic telemetry test that uses a standalone mock server at `test/integration/elastic_mock`. The meson build copies it to `$BUILD_PATH/test/elastic_mock` (with execute permissions), and `setup_suite.bash` adds `$BUILD_PATH/test` to `PATH` so it is available by name in all BATS tests. + +The script supports subcommands and is also useful for manual testing: + +```bash +# pick a free port, start the server (backgrounds itself with &) +port=$(elastic_mock free-port) +elastic_mock serve /tmp/cap.json "$port" & +elastic_mock wait-server "$port" # wait until accepting connections + +# send a request and inspect results +curl -s -X POST http://127.0.0.1:$port -d '{"name":"tool"}' +elastic_mock count /tmp/cap.json # → 1 +elastic_mock get /tmp/cap.json # pretty-print last record +elastic_mock assert /tmp/cap.json name tool # exits 0 +elastic_mock assert /tmp/cap.json name wrong # exits 1 + diagnostic + +# stop the server (or use `kill %1` / stop_elastic_mock in BATS) +elastic_mock kill /tmp/cap.json +``` + +In BATS tests the helper functions in `common.bash` wrap the common lifecycle: `start_elastic_mock CAPTURE_FILE PORT` (backgrounds `serve` and calls `wait-server`), `stop_elastic_mock` (kills by PID), and `wait_elastic_post CAPTURE_FILE [TIMEOUT]`. + +### Testing squashfs-mount + +The `squashfs-mount` helper requires setuid installation to test: + +```bash +# Set up staging path +export STAGE=$PWD/staging +export STAGING_PATH=$STAGE/usr/local + +# Build with squashfs-mount enabled +meson setup -Dtests=enabled -Dsquashfs_mount=true +meson compile + +# Install as root (for setuid bit) +sudo meson install --destdir=$STAGE --no-rebuild --skip-subprojects + +# Run tests (STAGING_PATH tells tests where to find the setuid binary) +./test/bats ./test/cli.bats +./test/bats ./test/squashfs-mount.bats +``` + +**IMPORTANT**: Never build with sudo (`sudo meson compile` or `sudo ninja`). Always build as normal user, then install with sudo. + +## Architecture + +### Source Structure + +- `src/cli/` - CLI command implementations (add_remove, build, completion, config, copy, delete, find, help, image, inspect, ls, pull, push, repo, run, start, status) +- `src/uenv/` - Core library shared between CLI and Slurm plugin + - Environment management (`env.h/cpp`, `uenv.h/cpp`) + - Repository/database operations (`repository.h/cpp`) + - Parsing (`parse.h/cpp`, `lex.h` in util) + - Mounting (`mount.h/cpp`) + - Meta data (`meta.h/cpp`) + - Container registry interaction (`oras.h/cpp`) + - Views (`view.h/cpp`) + - Telemetry (`telemetry.h/cpp`, `elastic.h/cpp`) + - Logging (`log.h/cpp`, `print.h/cpp`) + - Settings management (`settings.h/cpp`) +- `src/util/` - Utility libraries (color, curl, envvars, fs, lex, lustre, semver, shell, signal, strings, subprocess, toml) +- `src/site/` - Site-specific configuration (CSCS-specific logic) +- `src/slurm/` - Slurm plugin implementation +- `src/squashfs-mount/` - Setuid helper for mounting SquashFS images + +### Key Concepts + +**uenv_label**: Represents a uenv identifier with optional fields: +- Format: `name/version:tag@system%uarch` +- Example: `prgenv-gnu/24.11:v2@daint%gh200` + +**uenv_description**: Describes a uenv either by label or by filename, with optional mount point. + +**concrete_uenv**: Fully resolved uenv with paths (mount_path, sqfs_path, meta_path) and loaded metadata. + +**repository**: SQLite-backed database tracking available uenvs. Operations: query, add, remove, contains. + +**view**: Named environment configurations within a uenv (stored in env.json metadata). + +### Data Flow + +1. User provides uenv description (label or file path) +2. Parse into `uenv_description` using `parse.h` functions +3. Resolve to `concrete_uenv` (find squashfs image, mount location, metadata) +4. Load view from metadata (env.json) and configure update to the environment variables + - the CLI performs this by updating the environment variable store that is used to create `environ` for the call to exec in step 5 + - the Slurm plugin sets environment variables using setenv/getenv in the local context, and letting Slurm forward the environment to the remote context +5. Mount squashfs image at mount point + - the CLI does this by execing the squashfs_mount setuid helper, which in turn runs step 6 below + - the Slurm plugin performs the mount in the remote context as root before the daemon forks the MPI processes +6. Execute command with environment from view + +### Dependencies + +All dependencies are built as static libraries via meson wrap: +- CLI11 - command line parsing +- fmt - formatting library +- spdlog - logging +- nlohmann_json - JSON parsing +- sqlite3 - database +- libcurl - HTTP operations +- Catch2 - testing (when tests enabled) +- barkeep - progress indicators (header-only in `extern/`) + +The CLI build also downloads the ORAS binary for OCI registry operations. +ORAS is installed in `prefix/libexec` for runtime use by the CLI. + +## Development Notes + +### Code Style + +Always use braces `{}` on `if`, `for`, `while`, and other control flow statements, even for single-statement bodies. + +### Error Handling + +Use `util::expected` (similar to std::expected) for fallible operations. This is defined in `src/util/expected.h`. + +### Parsing + +The codebase has a custom lexer in `src/util/lex.h/cpp` for tokenizing inputs. +The shared parsing scaffolding — the `parse_error` type, the `PARSE` macro, and +the `parse_string` primitive — lives in `src/util/parse.h/cpp` (namespace `util`) +so that it can be reused by any module. +Parse functions return `util::expected`. The `uenv` parsers +are in `src/uenv/parse.h/cpp`; the OCI client has its own parsers in +`src/oci/parse.h/cpp` (see "Self-contained `src/oci`" below). +All inputs are parsed instead of using regex or simple string processing. +When we have to parse a new input type: +1. update the lexer (if needed) +2. add a `parse` function in the relevant `parse.h`/`parse.cpp` +3. write unit tests in the matching `test/unit/*.cpp` +4. run the unit tests, and repeat the process until they pass. + +### Self-contained `src/oci` + +The `src/oci/` module is a native OCI registry client written to replace the +external `oras` binary. It is intended to be reusable and independently testable, +so it must depend **only** on `src/util/` and external libraries (fmt, +nlohmann_json, libcurl, spdlog, zlib). + +`src/oci/` must **not** include or depend on `src/uenv/`, `src/site/`, or +`src/cli/`. Code that both `uenv` and `oci` need (for example the lexer and the +parsing scaffolding) lives in `src/util/` precisely so that `oci` never has to +reach up into `uenv`: + +- `src/util/lex.*` — the shared tokenizer. +- `src/util/parse.*` — `util::parse_error`, the `PARSE` macro, and + `util::parse_string`. + +Parsing in `src/oci` follows the same lexer-based idiom as the rest of the +codebase; JSON documents are parsed with nlohmann_json, while +`src/oci/parse.*` handles the character-level string types (digests, references, +URLs, the bearer challenge). + +Enforcement: this command must return nothing. + +```bash +grep -rn '#include <\(uenv\|site\|cli\)/' src/oci/ +``` + +### Environment Variables + +Use `envvars::state` to access environment variables (from `src/util/envvars.h`). Available as `settings.calling_environment` in most CLI commands. + +The tool follows the philosophy of not reading environment variables directly using `getenv`, instead we grab a read only copy of the environment at startup, stored in `settings.calling_environment`. +When calling `exec` to run a new command with a modified environment, we copy this initial state, modify it, then pass the modified copy to `exec`. + +### Logging + +Use spdlog for logging. Set verbosity via `settings.verbose`. Format output using fmt library. + +### File System Operations + +Prefer using functions from `src/util/fs.h` which provide expected-based error handling over raw std::filesystem operations. + +### Adding CLI Commands + +1. Create header/source in `src/cli/` (e.g., `foo.h`, `foo.cpp`) +2. Implement command function returning `int` (exit code) +3. Add source to `cli_src` array in `meson.build` +4. Register subcommand in `src/cli/uenv.cpp` main function using CLI11 +5. Add integration tests in `test/integration/cli.bats` +6. Add unit tests for any new library functions in `test/unit/` + +### Testing New Features + +- Add unit tests in `test/unit/` for library functions +- Add BATS integration tests in `test/integration/` for CLI behavior +- BATS tests use test data generated in `test/data/` and setup scripts in `test/setup/` diff --git a/meson.build b/meson.build index 91c3bf54..37a70052 100644 --- a/meson.build +++ b/meson.build @@ -64,7 +64,12 @@ lib_src = [ 'src/uenv/mount.cpp', 'src/oci/auth.cpp', 'src/oci/client.cpp', + 'src/oci/digest.cpp', + 'src/oci/manifest.cpp', + 'src/oci/parse.cpp', 'src/oci/pull.cpp', + 'src/oci/push.cpp', + 'src/oci/reference.cpp', 'src/uenv/oras.cpp', 'src/uenv/parse.cpp', 'src/uenv/print.cpp', @@ -78,6 +83,7 @@ lib_src = [ 'src/util/fs.cpp', 'src/util/lex.cpp', 'src/util/lustre.cpp', + 'src/util/parse.cpp', 'src/util/semver.cpp', 'src/util/sha256.cpp', 'src/util/shell.cpp', diff --git a/src/cli/copy.cpp b/src/cli/copy.cpp index 56023179..b867c67d 100644 --- a/src/cli/copy.cpp +++ b/src/cli/copy.cpp @@ -11,7 +11,8 @@ #include #include -#include +#include +#include #include #include #include @@ -67,12 +68,6 @@ int image_copy([[maybe_unused]] const image_copy_args& args, term::error("{}", c.error()); return 1; } - // copy still runs on oras; adapt the credentials at the legacy boundary. - std::optional oras_creds; - if (credentials) { - oras_creds = uenv::oras::credentials{credentials->username, - credentials->password}; - } uenv_nslabel src_label{}; if (const auto parse = parse_uenv_nslabel(args.src_uenv_description)) { @@ -178,13 +173,28 @@ int image_copy([[maybe_unused]] const image_copy_args& args, term::warn("the destination already exists and will be overwritten"); } - const auto rego_url = registry_cfg.url; - spdlog::debug("registry url: {}", rego_url); + // split the configured registry into a base URL + prefix, then build the + // source and destination OCI repository paths (the addresses oras used). + auto loc = oci::split_registry(registry_cfg.url); + if (!loc) { + term::error("invalid registry url: {}", loc.error().message()); + return 1; + } + const auto src_repo = oci::repository_path( + loc->prefix, src_label.nspace.value(), src_record.system, + src_record.uarch, src_record.name, src_record.version); + const auto dst_repo = oci::repository_path( + loc->prefix, dst_label.nspace.value(), dst_record.system, + dst_record.uarch, dst_record.name, dst_record.version); + const auto src_manifest = oci::digest::sha256(src_record.sha.string()); + spdlog::debug("oci copy: {} -> {} (tag {})", src_repo, dst_repo, + dst_record.tag); + if (auto result = - oras::copy(rego_url, src_label.nspace.value(), src_record, - dst_label.nspace.value(), dst_record, oras_creds); + oci::copy_image(loc->base, src_repo, dst_repo, src_manifest, + dst_record.tag, credentials); !result) { - term::error("unable to copy uenv.\n{}", result.error().message); + term::error("unable to copy uenv.\n{}", result.error()); return 1; } diff --git a/src/cli/pull.cpp b/src/cli/pull.cpp index a24dab79..5b3b88f1 100644 --- a/src/cli/pull.cpp +++ b/src/cli/pull.cpp @@ -179,27 +179,15 @@ int image_pull(const image_pull_args& args, const global_settings& settings) { // "jfrog.svc.cscs.ch/uenv") into a base URL and the repository prefix, // then build the OCI repository name the same way the oras address was // formed: /////. - std::string reg = registry_cfg.url; - for (const std::string scheme : {"https://", "http://"}) { - if (reg.rfind(scheme, 0) == 0) { - reg = reg.substr(scheme.size()); - break; - } - } - std::string host = reg; - std::string prefix; - if (auto slash = reg.find('/'); slash != std::string::npos) { - host = reg.substr(0, slash); - prefix = reg.substr(slash + 1); + auto loc = oci::split_registry(registry_cfg.url); + if (!loc) { + term::error("invalid registry url: {}", loc.error().message()); + return 1; } - const std::string registry_base = "https://" + host; + const std::string registry_base = loc->base; const std::string repository = - prefix.empty() - ? fmt::format("{}/{}/{}/{}/{}", nspace, record.system, - record.uarch, record.name, record.version) - : fmt::format("{}/{}/{}/{}/{}/{}", prefix, nspace, - record.system, record.uarch, record.name, - record.version); + oci::repository_path(loc->prefix, nspace, record.system, + record.uarch, record.name, record.version); spdlog::debug("oci pull: registry={} repository={}", registry_base, repository); @@ -213,20 +201,28 @@ int image_pull(const image_pull_args& args, const global_settings& settings) { } // identify the image by its manifest digest (record.sha). - const std::string manifest_ref = "sha256:" + record.sha.string(); + const auto image_digest = oci::digest::sha256(record.sha.string()); + const auto manifest_ref = oci::reference::digest(image_digest); try { - // the image manifest is needed for the squashfs layer; fetch once. - auto manifest = client->get_manifest(manifest_ref); - if (!manifest) { + // the image manifest is needed for the squashfs layer; fetch + parse + // once. + auto response = client->get_manifest(manifest_ref); + if (!response) { term::error("unable to fetch the image manifest:\n{}", + response.error()); + return 1; + } + auto manifest = oci::parse_manifest(response->body); + if (!manifest) { + term::error("unable to parse the image manifest:\n{}", manifest.error()); return 1; } if (pull_meta) { auto found = - oci::pull_meta(*client, manifest_ref, paths.store); + oci::pull_meta(*client, image_digest, paths.store); if (!found) { term::error("unable to pull meta data.\n{}", found.error()); return 1; diff --git a/src/cli/push.cpp b/src/cli/push.cpp index 3442a3f1..c3e6b35e 100644 --- a/src/cli/push.cpp +++ b/src/cli/push.cpp @@ -1,8 +1,9 @@ // vim: ts=4 sts=4 sw=4 et -#include #include +#include +#include #include #include #include @@ -11,7 +12,9 @@ #include #include -#include +#include +#include +#include #include #include #include @@ -72,12 +75,6 @@ int image_push([[maybe_unused]] const image_push_args& args, term::error("{}", c.error()); return 1; } - // push still runs on oras; adapt the credentials at the legacy boundary. - std::optional oras_creds; - if (credentials) { - oras_creds = uenv::oras::credentials{credentials->username, - credentials->password}; - } // parse and validate the destination (i.e. the label in the registry) uenv_nslabel dst_label{}; @@ -131,47 +128,73 @@ int image_push([[maybe_unused]] const image_push_args& args, } spdlog::info("image_push: squashfs {}", sqfs.value()); - try { - auto rego_url = registry_cfg.url; - spdlog::debug("registry url: {}", rego_url); + // split the configured registry "host/prefix" into a base URL + prefix and + // build the OCI repository path (the same address oras used). + auto loc = oci::split_registry(registry_cfg.url); + if (!loc) { + term::error("invalid registry url: {}", loc.error().message()); + return 1; + } + const auto& L = dst_label.label; + const auto repository = oci::repository_path( + loc->prefix, nspace, *L.system, *L.uarch, *L.name, *L.version); + spdlog::debug("oci push: registry={} repository={}", loc->base, repository); + + auto client = oci::client::create(loc->base, repository, credentials); + if (!client) { + term::error("unable to connect to the registry:\n{}", client.error()); + return 1; + } - // Push the SquashFS image - auto push_result = oras::push_tag(rego_url, nspace, dst_label.label, - sqfs->sqfs, oras_creds); + namespace bk = barkeep; + + // Push the SquashFS image. Ctrl-C is left with its default behaviour: the + // upload session is not committed until the manifest is PUT, so an interrupt + // leaves nothing referencing a partial blob. + std::optional image_digest; + { + auto spinner = bk::Animation({ + .message = fmt::format("pushing {} to registry", + sqfs->sqfs.filename().string()), + .style = bk::Ellipsis, + .no_tty = !isatty(fileno(stdout)), + }); + auto push_result = + oci::push_squashfs(*client, sqfs->sqfs, oci::reference::tag(*L.tag)); + spinner->done(); if (!push_result) { - term::error("unable to push uenv.\n{}", - push_result.error().message); + term::error("unable to push uenv.\n{}", push_result.error()); return 1; } - - // Check for metadata and push if available - if (sqfs->meta) { - spdlog::info("image_push: pushing metadata from {}", - sqfs->meta.value().string()); - - auto meta_result = - oras::push_meta(rego_url, nspace, dst_label.label, - sqfs->meta.value(), oras_creds); - if (!meta_result) { - spdlog::warn("unable to push metadata.\n{}", - meta_result.error().message); - term::warn("unable to push metadata.\n{}", - meta_result.error().message); - // Continue even if metadata push fails - } else { - spdlog::info("successfully pushed metadata"); - } + image_digest = *push_result; + } + spdlog::info("image_push: pushed image manifest {}", *image_digest); + + // Check for metadata and attach it if available. + if (sqfs->meta) { + spdlog::info("image_push: pushing metadata from {}", + sqfs->meta.value().string()); + auto spinner = bk::Animation({ + .message = "pushing metadata to registry", + .style = bk::Ellipsis, + .no_tty = !isatty(fileno(stdout)), + }); + auto meta_result = + oci::attach(*client, oci::reference::digest(*image_digest), + oci::artifact_type_meta, *sqfs->meta); + spinner->done(); + if (!meta_result) { + spdlog::warn("unable to push metadata.\n{}", meta_result.error()); + term::warn("unable to push metadata.\n{}", meta_result.error()); + // Continue even if metadata push fails. + } else { + spdlog::info("successfully pushed metadata"); } - - term::msg("successfully pushed {}", args.source); - term::msg("to {}", args.dest); - } catch (util::signal_exception& e) { - spdlog::info("user interrupted the upload with ctrl-c"); - - // reraise the signal - raise(e.signal); } + term::msg("successfully pushed {}", args.source); + term::msg("to {}", args.dest); + return 0; } // namespace uenv diff --git a/src/oci/auth.cpp b/src/oci/auth.cpp index 48f69059..5574f436 100644 --- a/src/oci/auth.cpp +++ b/src/oci/auth.cpp @@ -16,124 +16,16 @@ #include #include #include +#include namespace oci { -namespace { - -// split a scope value on spaces (the OCI spec permits a single scope parameter -// carrying a space-separated list). -void append_scopes(std::vector& out, std::string_view value) { - std::size_t i = 0; - while (i < value.size()) { - while (i < value.size() && - std::isspace(static_cast(value[i]))) { - ++i; - } - std::size_t start = i; - while (i < value.size() && - !std::isspace(static_cast(value[i]))) { - ++i; - } - if (i > start) { - out.emplace_back(value.substr(start, i - start)); - } - } -} - -} // namespace - // --- pure helpers (no network; unit-tested) ----------------------------- // Kept out of the public auth.h interface; tests redeclare these prototypes -// inside namespace oci::impl (see test/unit/oci_auth.cpp). +// inside namespace oci::impl (see test/unit/oci_auth.cpp). The bearer-challenge +// parser lives in src/oci/parse.cpp (oci::parse_bearer_challenge). namespace impl { -std::optional -parse_bearer_challenge(std::string_view v) { - std::string_view s = util::trim(v); - - // case-insensitive "Bearer" scheme, followed by whitespace - constexpr std::string_view scheme = "bearer"; - if (s.size() < scheme.size()) { - return std::nullopt; - } - for (std::size_t i = 0; i < scheme.size(); ++i) { - if (std::tolower(static_cast(s[i])) != scheme[i]) { - return std::nullopt; - } - } - s.remove_prefix(scheme.size()); - if (!s.empty() && !std::isspace(static_cast(s.front()))) { - return std::nullopt; // e.g. "BearerX" is not the Bearer scheme - } - s = util::trim(s); - - bearer_challenge challenge; - - std::size_t i = 0; - while (i < s.size()) { - // key = up to '=' or ',' - std::size_t key_start = i; - while (i < s.size() && s[i] != '=' && s[i] != ',') { - ++i; - } - std::string key = - util::to_lower(util::trim(s.substr(key_start, i - key_start))); - - std::string value; - if (i < s.size() && s[i] == '=') { - ++i; // consume '=' - if (i < s.size() && s[i] == '"') { - ++i; // consume opening quote - std::string buf; - while (i < s.size() && s[i] != '"') { - if (s[i] == '\\' && i + 1 < s.size()) { - buf.push_back(s[i + 1]); - i += 2; - } else { - buf.push_back(s[i]); - ++i; - } - } - if (i < s.size() && s[i] == '"') { - ++i; // consume closing quote - } - value = std::move(buf); - } else { - std::size_t val_start = i; - while (i < s.size() && s[i] != ',') { - ++i; - } - value = std::string( - util::trim(s.substr(val_start, i - val_start))); - } - } - - if (key == "realm") { - challenge.realm = std::move(value); - } else if (key == "service") { - challenge.service = std::move(value); - } else if (key == "scope") { - append_scopes(challenge.scopes, value); - } - - // advance past the separating comma and any whitespace - while (i < s.size() && s[i] != ',') { - ++i; - } - if (i < s.size() && s[i] == ',') { - ++i; - } - s = util::trim(s.substr(i)); - i = 0; - } - - if (challenge.realm.empty()) { - return std::nullopt; // realm is mandatory - } - return challenge; -} - std::string token_url(const bearer_challenge& challenge, const std::vector& scopes) { std::string url = challenge.realm; @@ -205,10 +97,11 @@ discover_challenge(const std::string& registry_url) { return util::unexpected{fmt::format( "{} returned 401 with no WWW-Authenticate header", req.url)}; } - auto challenge = impl::parse_bearer_challenge(*header); + auto challenge = parse_bearer_challenge(*header); if (!challenge) { return util::unexpected{fmt::format( - "could not parse WWW-Authenticate header: {}", *header)}; + "could not parse WWW-Authenticate header '{}': {}", *header, + challenge.error().message())}; } return *challenge; } diff --git a/src/oci/client.cpp b/src/oci/client.cpp index b3cb6d1c..e07a29fb 100644 --- a/src/oci/client.cpp +++ b/src/oci/client.cpp @@ -11,6 +11,7 @@ #include #include #include +#include namespace oci { @@ -23,10 +24,6 @@ namespace impl { std::string repository_scope(std::string_view repository, std::string_view actions); -std::string digest_string(const util::sha256_digest& d) { - return "sha256:" + d.hex(); -} - std::string blob_path(std::string_view repository, std::string_view digest) { return fmt::format("/v2/{}/blobs/{}", repository, digest); } @@ -105,12 +102,15 @@ std::optional> parse_referrers(std::string_view body) { if (!m.is_object()) { continue; } - descriptor d; - d.media_type = m.value("mediaType", ""); - d.digest = m.value("digest", ""); - d.size = m.value("size", std::size_t{0}); - if (auto a = m.find("artifactType"); - a != m.end() && a->is_string()) { + // a descriptor must carry a valid digest; skip malformed entries. + auto dg = digest::parse(m.value("digest", std::string{})); + if (!dg) { + continue; + } + descriptor d{.media_type = m.value("mediaType", std::string{}), + .digest = *dg, + .size = m.value("size", std::size_t{0})}; + if (auto a = m.find("artifactType"); a != m.end() && a->is_string()) { d.artifact_type = a->get(); } out.push_back(std::move(d)); @@ -120,6 +120,44 @@ std::optional> parse_referrers(std::string_view body) { } // namespace impl +// --- registry addressing helpers (pure) --------------------------------- + +util::expected +split_registry(std::string_view configured_url) { + // parse the configured value as a URL; the scheme, if any, is dropped and + // https is always used for the base. any port is preserved on the base. + auto u = parse_url(configured_url); + if (!u) { + return util::unexpected(u.error()); + } + + // the repository prefix is the URL path with its surrounding slashes trimmed. + std::string prefix = u->path; + if (!prefix.empty() && prefix.front() == '/') { + prefix.erase(prefix.begin()); + } + while (!prefix.empty() && prefix.back() == '/') { + prefix.pop_back(); + } + + std::string base = "https://" + u->host; + if (u->port) { + base += ':' + std::to_string(*u->port); + } + return registry_location{.base = std::move(base), .prefix = std::move(prefix)}; +} + +std::string repository_path(std::string_view prefix, std::string_view nspace, + std::string_view system, std::string_view uarch, + std::string_view name, std::string_view version) { + if (prefix.empty()) { + return fmt::format("{}/{}/{}/{}/{}", nspace, system, uarch, name, + version); + } + return fmt::format("{}/{}/{}/{}/{}/{}", prefix, nspace, system, uarch, name, + version); +} + // --- client ------------------------------------------------------------- util::expected @@ -146,8 +184,14 @@ util::expected client::token_for(bool write) { if (cache) { return *cache; } - auto scope = impl::repository_scope(repository_, write ? "pull,push" : "pull"); - auto token = fetch_token(challenge_, {scope}, creds_); + std::vector scopes; + scopes.push_back( + impl::repository_scope(repository_, write ? "pull,push" : "pull")); + // cross-repo blob mounts need pull scope on the source repository too. + for (const auto& r : extra_pull_scopes_) { + scopes.push_back(impl::repository_scope(r, "pull")); + } + auto token = fetch_token(challenge_, scopes, creds_); if (!token) { return util::unexpected{token.error()}; } @@ -155,14 +199,17 @@ util::expected client::token_for(bool write) { return *cache; } -util::expected -client::blob_exists(const std::string& digest) { +void client::add_pull_scope(std::string repository) { + extra_pull_scopes_.push_back(std::move(repository)); +} + +util::expected client::blob_exists(const digest& d) { auto token = token_for(false); if (!token) { return util::unexpected{token.error()}; } util::curl::request req; - req.url = registry_url_ + impl::blob_path(repository_, digest); + req.url = registry_url_ + impl::blob_path(repository_, d.string()); req.method = util::curl::http_method::head; req.bearer_token = *token; @@ -176,18 +223,17 @@ client::blob_exists(const std::string& digest) { if (resp->status == 404) { return false; } - return util::unexpected{fmt::format( - "unexpected status {} for HEAD {}", resp->status, digest)}; + return util::unexpected{fmt::format("unexpected status {} for HEAD {}", + resp->status, d.string())}; } -util::expected -client::get_blob(const std::string& digest) { +util::expected client::get_blob(const digest& d) { auto token = token_for(false); if (!token) { return util::unexpected{token.error()}; } util::curl::request req; - req.url = registry_url_ + impl::blob_path(repository_, digest); + req.url = registry_url_ + impl::blob_path(repository_, d.string()); req.bearer_token = *token; // registries 307-redirect blob downloads to backing storage. req.follow_redirects = true; @@ -198,14 +244,14 @@ client::get_blob(const std::string& digest) { } if (resp->status != 200) { return util::unexpected{fmt::format( - "failed to fetch blob {} (status {})", digest, resp->status)}; + "failed to fetch blob {} (status {})", d.string(), resp->status)}; } return resp->body; } util::expected client::get_blob_to_file( - const std::string& digest, const std::filesystem::path& file, + const digest& d, const std::filesystem::path& file, std::function progress, std::function should_abort) { auto token = token_for(false); @@ -213,7 +259,7 @@ client::get_blob_to_file( return util::unexpected{token.error()}; } util::curl::request req; - req.url = registry_url_ + impl::blob_path(repository_, digest); + req.url = registry_url_ + impl::blob_path(repository_, d.string()); req.bearer_token = *token; req.follow_redirects = true; req.download_file = file; @@ -226,19 +272,19 @@ client::get_blob_to_file( } if (resp->status != 200) { return util::unexpected{fmt::format( - "failed to fetch blob {} (status {})", digest, resp->status)}; + "failed to fetch blob {} (status {})", d.string(), resp->status)}; } return {}; } util::expected -client::get_manifest(const std::string& reference) { +client::get_manifest(const reference& ref) { auto token = token_for(false); if (!token) { return util::unexpected{token.error()}; } util::curl::request req; - req.url = registry_url_ + impl::manifest_path(repository_, reference); + req.url = registry_url_ + impl::manifest_path(repository_, ref.string()); req.bearer_token = *token; req.header_lines = {fmt::format("Accept: {}", media_type_manifest), fmt::format("Accept: {}", media_type_index)}; @@ -249,12 +295,16 @@ client::get_manifest(const std::string& reference) { } if (resp->status != 200) { return util::unexpected{fmt::format( - "failed to fetch manifest {} (status {})", reference, + "failed to fetch manifest {} (status {})", ref.string(), resp->status)}; } manifest_response m; m.body = std::move(resp->body); - m.digest = resp->headers.get("docker-content-digest").value_or(""); + if (auto header = resp->headers.get("docker-content-digest")) { + if (auto d = digest::parse(*header)) { + m.digest = *d; + } + } m.media_type = resp->headers.get("content-type").value_or(""); return m; } @@ -284,13 +334,13 @@ util::expected, std::string> client::list_tags() { } util::expected, std::string> -client::referrers(const std::string& digest) { +client::referrers(const digest& d) { auto token = token_for(false); if (!token) { return util::unexpected{token.error()}; } util::curl::request req; - req.url = registry_url_ + impl::referrers_path(repository_, digest); + req.url = registry_url_ + impl::referrers_path(repository_, d.string()); req.bearer_token = *token; auto resp = util::curl::perform(req); @@ -299,7 +349,7 @@ client::referrers(const std::string& digest) { } if (resp->status != 200) { return util::unexpected{fmt::format( - "failed to fetch referrers of {} (status {})", digest, + "failed to fetch referrers of {} (status {})", d.string(), resp->status)}; } auto refs = impl::parse_referrers(resp->body); @@ -360,45 +410,76 @@ do_put_blob(const std::string& registry_url, const std::string& uploads, } // namespace +util::expected +client::mount_blob(const digest& d, const std::string& from_repository) { + if (auto exists = blob_exists(d); exists && *exists) { + return true; + } + auto token = token_for(true); + if (!token) { + return util::unexpected{token.error()}; + } + util::curl::request post; + post.url = registry_url_ + impl::uploads_path(repository_) + + "?mount=" + d.string() + "&from=" + from_repository; + post.method = util::curl::http_method::post; + post.bearer_token = *token; + + auto resp = util::curl::perform(post); + if (!resp) { + return util::unexpected{resp.error().message}; + } + // 201: the blob was mounted. 202: the registry declined and opened an upload + // session instead (caller must copy the blob the slow way). + if (resp->status == 201) { + return true; + } + if (resp->status == 202) { + return false; + } + return util::unexpected{fmt::format( + "failed to mount blob {} from {} (status {}): {}", d.string(), + from_repository, resp->status, resp->body)}; +} + util::expected -client::put_blob(const std::string& digest, - const std::filesystem::path& file) { - if (auto exists = blob_exists(digest); exists && *exists) { - spdlog::trace("oci::put_blob {} already present", digest); +client::put_blob(const digest& d, const std::filesystem::path& file) { + if (auto exists = blob_exists(d); exists && *exists) { + spdlog::trace("oci::put_blob {} already present", d.string()); return {}; } auto token = token_for(true); if (!token) { return util::unexpected{token.error()}; } - return do_put_blob(registry_url_, impl::uploads_path(repository_), *token, digest, - [&file](util::curl::request& r) { - r.upload_file = file; - }); + return do_put_blob(registry_url_, impl::uploads_path(repository_), *token, + d.string(), + [&file](util::curl::request& r) { r.upload_file = file; }); } util::expected -client::put_blob_bytes(const std::string& digest, const std::string& data) { - if (auto exists = blob_exists(digest); exists && *exists) { +client::put_blob_bytes(const digest& d, const std::string& data) { + if (auto exists = blob_exists(d); exists && *exists) { return {}; } auto token = token_for(true); if (!token) { return util::unexpected{token.error()}; } - return do_put_blob(registry_url_, impl::uploads_path(repository_), *token, digest, + return do_put_blob(registry_url_, impl::uploads_path(repository_), *token, + d.string(), [&data](util::curl::request& r) { r.body = data; }); } -util::expected -client::put_manifest(const std::string& reference, const std::string& body, +util::expected +client::put_manifest(const reference& ref, const std::string& body, std::string_view media_type) { auto token = token_for(true); if (!token) { return util::unexpected{token.error()}; } util::curl::request req; - req.url = registry_url_ + impl::manifest_path(repository_, reference); + req.url = registry_url_ + impl::manifest_path(repository_, ref.string()); req.method = util::curl::http_method::put; req.bearer_token = *token; req.body = body; @@ -410,10 +491,10 @@ client::put_manifest(const std::string& reference, const std::string& body, } if (resp->status != 201) { return util::unexpected{fmt::format( - "failed to put manifest {} (status {}): {}", reference, + "failed to put manifest {} (status {}): {}", ref.string(), resp->status, resp->body)}; } - return resp->headers.get("docker-content-digest").value_or(""); + return {}; } } // namespace oci diff --git a/src/oci/client.h b/src/oci/client.h index f1510191..6b3869ad 100644 --- a/src/oci/client.h +++ b/src/oci/client.h @@ -10,7 +10,10 @@ #include #include +#include +#include #include +#include #include namespace oci { @@ -23,22 +26,46 @@ inline constexpr std::string_view media_type_index = inline constexpr std::string_view media_type_octet_stream = "application/octet-stream"; -// An OCI content descriptor (a manifest entry / referrer record). +// An OCI content descriptor (a manifest entry / referrer record). A descriptor +// is not default-constructible: it must carry a valid `digest`, so a descriptor +// value is always well-formed. struct descriptor { std::string media_type; - std::string digest; // "sha256:" + oci::digest digest; std::size_t size = 0; std::optional artifact_type; + // inline base64 content (`data`), present on the empty config descriptor. + std::optional data; friend bool operator==(const descriptor&, const descriptor&) = default; }; +// A registry address split into an https base URL and a repository path prefix. +struct registry_location { + std::string base; // e.g. "https://jfrog.svc.cscs.ch" + std::string prefix; // e.g. "uenv" (may be empty) +}; + +// Split a configured registry URL ("host", "host/prefix", or scheme-prefixed) +// into an https base URL and a repository prefix. The scheme, if any, is dropped +// and https is always used for the base (matching how pull resolves it). +util::expected +split_registry(std::string_view configured_url); + +// Build the OCI repository path for a uenv, e.g. +// "/////" (the prefix segment is +// omitted when empty). This mirrors the address oras formed. +std::string repository_path(std::string_view prefix, std::string_view nspace, + std::string_view system, std::string_view uarch, + std::string_view name, std::string_view version); + // The result of fetching a manifest: the raw bytes plus the registry-reported // digest and media type. The bytes are what must be re-digested locally to // confirm identity. struct manifest_response { std::string body; - std::string digest; // value of the Docker-Content-Digest header + // value of the Docker-Content-Digest header, when present and well-formed. + std::optional digest; std::string media_type; }; @@ -57,11 +84,10 @@ class client { std::optional creds = std::nullopt); // does a blob exist? HEAD; 200 -> true, 404 -> false. - util::expected blob_exists(const std::string& digest); + util::expected blob_exists(const digest& d); // fetch a blob's bytes (follows the 307 redirect to backing storage). - util::expected - get_blob(const std::string& digest); + util::expected get_blob(const digest& d); // stream a blob straight to a file, never holding it in memory (for the // multi-GB squashfs layer). follows the 307 redirect to backing storage. @@ -69,37 +95,49 @@ class client { // an optional abort predicate, polled during transfer, cancels it. util::expected get_blob_to_file( - const std::string& digest, const std::filesystem::path& file, + const digest& d, const std::filesystem::path& file, std::function progress = {}, std::function should_abort = {}); // fetch a manifest by tag or digest. util::expected - get_manifest(const std::string& reference); + get_manifest(const reference& ref); // list the repository's tags. util::expected, std::string> list_tags(); - // list artifacts that refer to `digest` (replaces `oras discover`). + // list artifacts that refer to `d` (replaces `oras discover`). util::expected, std::string> - referrers(const std::string& digest); + referrers(const digest& d); + + // attempt a cross-repository blob mount from `from_repository` into this + // client's repository (POST uploads/?mount=&from=). returns true if the + // registry mounted the blob (201), false if it declined and wants a full + // upload instead (202) — the caller should then fall back to copying. no-op + // (returns true) if the blob already exists here. + util::expected + mount_blob(const digest& d, const std::string& from_repository); // upload a blob via the monolithic POST-then-PUT handshake. the file is // streamed from disk (not held in memory), so large squashfs layers are // fine. no-op if the blob already exists. util::expected - put_blob(const std::string& digest, const std::filesystem::path& file); + put_blob(const digest& d, const std::filesystem::path& file); // upload an in-memory blob (config blobs, small payloads). util::expected - put_blob_bytes(const std::string& digest, const std::string& data); + put_blob_bytes(const digest& d, const std::string& data); - // PUT a manifest under `reference` (tag or digest). returns the registry's - // Docker-Content-Digest (the canonical id). - util::expected - put_manifest(const std::string& reference, const std::string& body, + // PUT a manifest under `ref` (tag or digest). + util::expected + put_manifest(const reference& ref, const std::string& body, std::string_view media_type = media_type_manifest); + // grant this client pull access to an additional repository, so its tokens + // carry the scope a cross-repo blob mount needs (see mount_blob). must be + // called before the first operation that fetches a token. + void add_pull_scope(std::string repository); + const std::string& registry_url() const { return registry_url_; } @@ -119,6 +157,8 @@ class client { bearer_challenge challenge_; std::optional pull_token_; std::optional push_token_; + // extra repositories to request pull scope for (cross-repo mount). + std::vector extra_pull_scopes_; }; } // namespace oci diff --git a/src/oci/digest.cpp b/src/oci/digest.cpp new file mode 100644 index 00000000..d55bd7ca --- /dev/null +++ b/src/oci/digest.cpp @@ -0,0 +1,45 @@ +#include +#include +#include +#include + +#include + +#include +#include + +namespace oci { + +namespace { + +bool is_lower_hex(std::string_view s) { + for (char c : s) { + const bool ok = (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'); + if (!ok) { + return false; + } + } + return true; +} + +} // namespace + +digest digest::from_sha256(const util::sha256_digest& d) { + return digest{"sha256", d.hex()}; +} + +digest digest::sha256(std::string hex) { + assert(hex.size() == 64 && is_lower_hex(hex) && + "digest::sha256 requires a 64-char lowercase-hex string"); + return digest{"sha256", std::move(hex)}; +} + +util::expected digest::parse(std::string_view text) { + return oci::parse_digest(text); +} + +std::string digest::string() const { + return algorithm_ + ":" + hex_; +} + +} // namespace oci diff --git a/src/oci/digest.h b/src/oci/digest.h new file mode 100644 index 00000000..57e8c433 --- /dev/null +++ b/src/oci/digest.h @@ -0,0 +1,69 @@ +#pragma once + +#include +#include + +#include +#include +#include + +namespace oci { + +class digest; +// the lexer-based parser (src/oci/parse.cpp) constructs digests directly. +util::expected parse_digest(std::string_view); + +// An OCI content digest: an algorithm plus a lowercase-hex value, rendered as +// ":" (e.g. "sha256:abc..."). A `digest` is always +// syntactically valid — it can only be obtained by parsing (which validates the +// algorithm and hex) or by promoting a computed hash — so callers never juggle +// bare hex against prefixed strings or a tag against a digest. +class digest { + public: + // promote a computed sha-256 hash to a digest. + static digest from_sha256(const util::sha256_digest& d); + + // build a sha-256 digest from a known-valid 64-char lowercase-hex string + // (e.g. a uenv record's sha). the precondition is asserted; use parse() for + // input that might be malformed. + static digest sha256(std::string hex); + + // parse ":". rejects an unknown algorithm, and a value whose + // length does not match the algorithm or that is not lowercase hex. a thin + // forwarder to oci::parse_digest. + static util::expected parse(std::string_view text); + + const std::string& algorithm() const { + return algorithm_; + } + const std::string& hex() const { + return hex_; + } + // the canonical ":" string. + std::string string() const; + + friend bool operator==(const digest&, const digest&) = default; + friend util::expected + parse_digest(std::string_view); + + private: + digest(std::string algorithm, std::string hex) + : algorithm_(std::move(algorithm)), hex_(std::move(hex)) { + } + std::string algorithm_; + std::string hex_; +}; + +} // namespace oci + +#include +template <> class fmt::formatter { + public: + constexpr auto parse(format_parse_context& ctx) { + return ctx.end(); + } + template + auto format(oci::digest const& d, FmtContext& ctx) const { + return fmt::format_to(ctx.out(), "{}", d.string()); + } +}; diff --git a/src/oci/manifest.cpp b/src/oci/manifest.cpp new file mode 100644 index 00000000..5aaab601 --- /dev/null +++ b/src/oci/manifest.cpp @@ -0,0 +1,212 @@ +#include +#include + +#include +#include + +#include +#include +#include + +namespace oci { + +descriptor empty_config_descriptor() { + return descriptor{.media_type = std::string{media_type_empty}, + .digest = digest::sha256(std::string{empty_config_hex}), + .size = empty_config_body.size(), + .artifact_type = std::nullopt, + .data = std::string{empty_config_data}}; +} + +std::optional manifest_layer::title() const { + if (auto it = annotations.find(std::string{annotation_title}); + it != annotations.end()) { + return it->second; + } + return std::nullopt; +} + +bool manifest_layer::wants_unpack() const { + auto it = annotations.find(std::string{annotation_unpack}); + return it != annotations.end() && it->second == "true"; +} + +const manifest_layer* +manifest::find_layer_by_title(std::string_view title) const { + for (const auto& l : layers) { + if (auto t = l.title(); t && *t == title) { + return &l; + } + } + return nullptr; +} + +const manifest_layer* manifest::find_unpack_layer() const { + for (const auto& l : layers) { + if (l.wants_unpack()) { + return &l; + } + } + return nullptr; +} + +// --- serialization ----------------------------------------------------------- + +namespace { + +nlohmann::ordered_json descriptor_json(const descriptor& d) { + nlohmann::ordered_json j; + j["mediaType"] = d.media_type; + j["digest"] = d.digest.string(); + j["size"] = d.size; + if (d.artifact_type) { + j["artifactType"] = *d.artifact_type; + } + if (d.data) { + j["data"] = *d.data; + } + return j; +} + +nlohmann::ordered_json annotations_json( + const std::map& annotations) { + nlohmann::ordered_json j; + // std::map iterates in sorted key order, matching oras. + for (const auto& [k, v] : annotations) { + j[k] = v; + } + return j; +} + +} // namespace + +std::string serialize_manifest(const manifest& m) { + nlohmann::ordered_json j; + j["schemaVersion"] = 2; + j["mediaType"] = m.media_type; + if (!m.artifact_type.empty()) { + j["artifactType"] = m.artifact_type; + } + j["config"] = descriptor_json(m.config); + + auto layers = nlohmann::ordered_json::array(); + for (const auto& l : m.layers) { + nlohmann::ordered_json lj; + lj["mediaType"] = l.media_type; + lj["digest"] = l.digest.string(); + lj["size"] = l.size; + if (!l.annotations.empty()) { + lj["annotations"] = annotations_json(l.annotations); + } + layers.push_back(std::move(lj)); + } + j["layers"] = std::move(layers); + + if (m.subject) { + j["subject"] = descriptor_json(*m.subject); + } + if (!m.annotations.empty()) { + j["annotations"] = annotations_json(m.annotations); + } + + return j.dump(); +} + +std::string serialize_index(const std::vector& manifests) { + nlohmann::ordered_json j; + j["schemaVersion"] = 2; + j["mediaType"] = media_type_index; + auto arr = nlohmann::ordered_json::array(); + for (const auto& d : manifests) { + arr.push_back(descriptor_json(d)); + } + j["manifests"] = std::move(arr); + return j.dump(); +} + +// --- parsing ----------------------------------------------------------------- + +namespace { + +util::expected +parse_descriptor(const nlohmann::json& j) { + auto dg = digest::parse(j.value("digest", std::string{})); + if (!dg) { + return util::unexpected{dg.error().message()}; + } + descriptor d{.media_type = j.value("mediaType", std::string{}), + .digest = *dg, + .size = j.value("size", std::size_t{0})}; + if (auto a = j.find("artifactType"); a != j.end() && a->is_string()) { + d.artifact_type = a->get(); + } + if (auto dt = j.find("data"); dt != j.end() && dt->is_string()) { + d.data = dt->get(); + } + return d; +} + +std::map +parse_annotations(const nlohmann::json& j) { + std::map out; + if (auto a = j.find("annotations"); a != j.end() && a->is_object()) { + for (const auto& [k, v] : a->items()) { + if (v.is_string()) { + out[k] = v.get(); + } + } + } + return out; +} + +} // namespace + +util::expected parse_manifest(std::string_view body) { + auto j = nlohmann::json::parse(body, nullptr, /*allow_exceptions=*/false); + if (j.is_discarded() || !j.is_object()) { + return util::unexpected{"could not parse manifest JSON"}; + } + + manifest m; + m.media_type = j.value("mediaType", std::string{media_type_manifest}); + m.artifact_type = j.value("artifactType", std::string{}); + + if (auto c = j.find("config"); c != j.end() && c->is_object()) { + auto d = parse_descriptor(*c); + if (!d) { + return util::unexpected{ + fmt::format("invalid config descriptor: {}", d.error())}; + } + m.config = *d; + } + + if (auto ls = j.find("layers"); ls != j.end() && ls->is_array()) { + for (const auto& l : *ls) { + auto dg = digest::parse(l.value("digest", std::string{})); + if (!dg) { + return util::unexpected{ + fmt::format("invalid layer digest: {}", dg.error().message())}; + } + manifest_layer layer{.media_type = l.value("mediaType", + std::string{}), + .digest = *dg, + .size = l.value("size", std::size_t{0}), + .annotations = parse_annotations(l)}; + m.layers.push_back(std::move(layer)); + } + } + + if (auto s = j.find("subject"); s != j.end() && s->is_object()) { + auto d = parse_descriptor(*s); + if (!d) { + return util::unexpected{ + fmt::format("invalid subject descriptor: {}", d.error())}; + } + m.subject = *d; + } + + m.annotations = parse_annotations(j); + return m; +} + +} // namespace oci diff --git a/src/oci/manifest.h b/src/oci/manifest.h new file mode 100644 index 00000000..5592c36d --- /dev/null +++ b/src/oci/manifest.h @@ -0,0 +1,101 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace oci { + +// --- media types + the canonical empty config -------------------------------- + +// oras-style artifact manifests carry an "empty" JSON config object. This is the +// well-known descriptor for the two-byte body "{}". +inline constexpr std::string_view media_type_empty = + "application/vnd.oci.empty.v1+json"; +inline constexpr std::string_view empty_config_body = "{}"; +// the sha-256 of "{}" and its base64 encoding. +inline constexpr std::string_view empty_config_hex = + "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"; +inline constexpr std::string_view empty_config_data = "e30="; + +// layer media types used by uenv artifacts. +inline constexpr std::string_view media_type_layer_tar = + "application/vnd.oci.image.layer.v1.tar"; +inline constexpr std::string_view media_type_layer_targz = + "application/vnd.oci.image.layer.v1.tar+gzip"; + +// artifact types. +inline constexpr std::string_view artifact_type_squashfs = + "application/x-squashfs"; +inline constexpr std::string_view artifact_type_meta = "uenv/meta"; + +// annotation keys. +inline constexpr std::string_view annotation_title = + "org.opencontainers.image.title"; +inline constexpr std::string_view annotation_created = + "org.opencontainers.image.created"; +inline constexpr std::string_view annotation_unpack = + "io.deis.oras.content.unpack"; +inline constexpr std::string_view annotation_content_digest = + "io.deis.oras.content.digest"; + +// the canonical empty-config descriptor (inline "{}" with base64 `data`). +descriptor empty_config_descriptor(); + +// --- manifest model ---------------------------------------------------------- + +// A layer entry in an image manifest: a blob descriptor plus its annotations. +// annotations are held in a sorted map so serialization is deterministic and +// matches oras (which serializes Go maps with sorted keys). +struct manifest_layer { + std::string media_type; + oci::digest digest; + std::size_t size = 0; + std::map annotations; + + // value of org.opencontainers.image.title, if present. + std::optional title() const; + // true if annotated io.deis.oras.content.unpack == "true". + bool wants_unpack() const; +}; + +// A full OCI image manifest (application/vnd.oci.image.manifest.v1+json). This +// is both the input to serialize_manifest (write) and the result of +// parse_manifest (read). New instances default to the empty config, since that +// is what every artifact uenv writes uses. +struct manifest { + std::string media_type = std::string{media_type_manifest}; + std::string artifact_type; + descriptor config = empty_config_descriptor(); + std::vector layers; + std::optional subject; + std::map annotations; + + // find the layer with org.opencontainers.image.title == title, or nullptr. + const manifest_layer* find_layer_by_title(std::string_view title) const; + // find the first layer marked for unpacking, or nullptr. + const manifest_layer* find_unpack_layer() const; +}; + +// Serialize `m` to a compact OCI image-manifest JSON document, byte-for-byte in +// the same shape oras produces: schemaVersion, mediaType, artifactType, config, +// layers, optional subject, annotations. +std::string serialize_manifest(const manifest& m); + +// Parse an OCI image-manifest JSON document. Fails if the JSON is malformed or a +// descriptor digest is invalid. +util::expected parse_manifest(std::string_view body); + +// Serialize an OCI image index (application/vnd.oci.image.index.v1+json) listing +// `manifests`. Used for the referrers-tag fallback on registries that do not +// implement the Referrers API. +std::string serialize_index(const std::vector& manifests); + +} // namespace oci diff --git a/src/oci/parse.cpp b/src/oci/parse.cpp new file mode 100644 index 00000000..7ee22b19 --- /dev/null +++ b/src/oci/parse.cpp @@ -0,0 +1,508 @@ +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +namespace oci { + +// the parsing scaffolding lives in util; use the unqualified name here (as +// src/uenv/parse.cpp does) so the parse_error{...} constructions read cleanly. +using util::parse_error; + +namespace { + +// --- token predicates --------------------------------------------------- + +// alphanumeric run: the lexer splits e.g. "sha256" into symbol("sha") + +// integer("256") and a hex string into alternating symbol/integer tokens, so +// both a digest algorithm and its hex value are runs of these two kinds. +bool is_alnum_tok(lex::tok t) { + return t == lex::tok::symbol || t == lex::tok::integer; +} + +// OCI tag grammar body: [a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}. underscore is part of +// a symbol token. +bool is_tag_tok(lex::tok t) { + return t == lex::tok::symbol || t == lex::tok::integer || + t == lex::tok::dot || t == lex::tok::dash; +} + +// the leading token of a URL scheme (ALPHA), and the continuation tokens +// (ALPHA / DIGIT / "+" / "-" / "."). +bool is_scheme_tok(lex::tok t) { + return t == lex::tok::symbol || t == lex::tok::integer || + t == lex::tok::plus || t == lex::tok::dash || t == lex::tok::dot; +} + +// a URL reg-name host component (unreserved + pct-encoded); we stop at ':', '/', +// '?', '#', whitespace or end. +bool is_regname_tok(lex::tok t) { + return t == lex::tok::symbol || t == lex::tok::integer || + t == lex::tok::dash || t == lex::tok::dot || t == lex::tok::tilde || + t == lex::tok::percent; +} + +// the body of an IPv6 literal between '[' and ']'. +bool is_ipv6_tok(lex::tok t) { + return t == lex::tok::symbol || t == lex::tok::integer || + t == lex::tok::colon || t == lex::tok::dot || t == lex::tok::percent; +} + +// a URL path segment run (pchar plus '/'): everything up to a query, fragment, +// whitespace or end. +bool is_path_tok(lex::tok t) { + switch (t) { + case lex::tok::question: + case lex::tok::hash: + case lex::tok::whitespace: + case lex::tok::end: + case lex::tok::error: + return false; + default: + return true; + } +} + +// a query run: everything up to a fragment, whitespace or end. +bool is_query_tok(lex::tok t) { + switch (t) { + case lex::tok::hash: + case lex::tok::whitespace: + case lex::tok::end: + case lex::tok::error: + return false; + default: + return true; + } +} + +// a fragment run: everything up to whitespace or end. +bool is_fragment_tok(lex::tok t) { + switch (t) { + case lex::tok::whitespace: + case lex::tok::end: + case lex::tok::error: + return false; + default: + return true; + } +} + +// an unquoted auth-parameter value: everything up to a ',' or end. +bool is_unquoted_value_tok(lex::tok t) { + return t != lex::tok::comma && t != lex::tok::end; +} + +// the hex length expected for a recognised algorithm, or 0 if unrecognised. +std::size_t hex_length(std::string_view algorithm) { + if (algorithm == "sha256") { + return 64; + } + if (algorithm == "sha512") { + return 128; + } + return 0; +} + +bool is_lower_hex(std::string_view s) { + for (char c : s) { + const bool ok = (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'); + if (!ok) { + return false; + } + } + return true; +} + +// require the lexer to be at end-of-input, otherwise build a "trailing input" +// parse_error for `what`. +util::expected expect_end(lex::lexer& L, + std::string_view what) { + if (L != lex::tok::end) { + const auto t = L.peek(); + return util::unexpected(parse_error{ + L.string(), fmt::format("unexpected '{}' in {}", t.spelling, what), + t}); + } + return {}; +} + +} // namespace + +util::expected parse_digest(std::string_view text) { + const auto s = util::strip(text); + lex::lexer L(s); + + const auto algo_tok = L.peek(); + auto algorithm = util::parse_string(L, "digest algorithm", is_alnum_tok); + if (!algorithm) { + return util::unexpected(algorithm.error()); + } + if (L != lex::tok::colon) { + const auto t = L.peek(); + return util::unexpected(parse_error{ + L.string(), "expected ':' separating the algorithm and value", t}); + } + L.next(); // consume ':' + + const auto hex_tok = L.peek(); + auto hex = util::parse_string(L, "digest value", is_alnum_tok); + if (!hex) { + return util::unexpected(hex.error()); + } + if (auto e = expect_end(L, "digest"); !e) { + return util::unexpected(e.error()); + } + + const auto want = hex_length(*algorithm); + if (want == 0) { + return util::unexpected(parse_error{ + L.string(), fmt::format("unsupported digest algorithm '{}'", *algorithm), + algo_tok}); + } + if (hex->size() != want) { + return util::unexpected(parse_error{ + L.string(), + fmt::format("expected {} hex characters, got {}", want, hex->size()), + hex_tok}); + } + if (!is_lower_hex(*hex)) { + return util::unexpected(parse_error{ + L.string(), "the digest value is not lowercase hex", hex_tok}); + } + + // parse_digest is a friend of digest, so it can use the private constructor. + return digest{std::move(*algorithm), std::move(*hex)}; +} + +util::expected +parse_reference(std::string_view text) { + // a digest is unambiguous (it contains ':'); try it first. + if (auto d = parse_digest(text)) { + return reference::digest(*d); + } + + const auto s = util::strip(text); + lex::lexer L(s); + + const auto start = L.peek(); + // the OCI tag grammar forbids a leading '.' or '-'. + if (L != lex::tok::symbol && + L != lex::tok::integer) { + return util::unexpected(parse_error{ + L.string(), + fmt::format("'{}' is not a valid tag or digest reference", s), + start}); + } + auto tag = util::parse_string(L, "tag", is_tag_tok); + if (!tag) { + return util::unexpected(tag.error()); + } + if (auto e = expect_end(L, "reference"); !e) { + return util::unexpected(e.error()); + } + if (tag->size() > 128) { + return util::unexpected(parse_error{ + L.string(), "a tag may be at most 128 characters", start}); + } + return reference::tag(std::move(*tag)); +} + +std::string url::string() const { + std::string out; + if (!scheme.empty()) { + out += scheme; + out += "://"; + } + if (!userinfo.empty()) { + out += userinfo; + out += '@'; + } + out += host; + if (port) { + out += ':'; + out += std::to_string(*port); + } + out += path; + if (!query.empty()) { + out += '?'; + out += query; + } + if (!fragment.empty()) { + out += '#'; + out += fragment; + } + return out; +} + +util::expected parse_url(std::string_view text) { + const auto s = util::strip(text); + lex::lexer L(s); + url u; + + // optional scheme: "://". consume a scheme-shaped run, and only + // accept it as a scheme if it is followed by "://"; otherwise rewind (a bare + // "host/prefix" or "host:port" begins with the same tokens). + { + const unsigned start = L.peek().loc; + std::string scheme; + while (is_scheme_tok(L.current_kind())) { + scheme += L.next().spelling; + } + if (!scheme.empty() && L == lex::tok::colon && + L.peek(1) == lex::tok::slash && L.peek(2) == lex::tok::slash) { + L.next(); // ':' + L.next(); // '/' + L.next(); // '/' + u.scheme = std::move(scheme); + } else { + L.seek(start); + } + } + + // optional userinfo: present only when an '@' occurs before the first + // '/', '?', '#' or end. + { + bool has_userinfo = false; + for (unsigned k = 0;; ++k) { + const auto t = L.peek(k); + if (t == lex::tok::slash || t == lex::tok::question || + t == lex::tok::hash || t == lex::tok::end) { + break; + } + if (t == lex::tok::at) { + has_userinfo = true; + break; + } + } + if (has_userinfo) { + std::string userinfo; + while (L != lex::tok::at) { + userinfo += L.next().spelling; + } + L.next(); // consume '@' + u.userinfo = std::move(userinfo); + } + } + + // host: an IPv6 literal in brackets, or a reg-name. + const auto host_tok = L.peek(); + if (L == lex::tok::lbracket) { + std::string host = std::string{L.next().spelling}; // '[' + while (is_ipv6_tok(L.current_kind())) { + host += L.next().spelling; + } + if (L != lex::tok::rbracket) { + return util::unexpected(parse_error{ + L.string(), "unterminated IPv6 host literal", host_tok}); + } + host += L.next().spelling; // ']' + u.host = std::move(host); + } else { + auto host = util::parse_string(L, "host", is_regname_tok); + if (!host) { + return util::unexpected(parse_error{ + L.string(), "the url has no host", host_tok}); + } + u.host = std::move(*host); + } + + // optional port. + if (L == lex::tok::colon) { + L.next(); // consume ':' + const auto port_tok = L.peek(); + if (L != lex::tok::integer) { + return util::unexpected(parse_error{ + L.string(), "expected a port number after ':'", port_tok}); + } + const auto digits = L.next().spelling; + std::uint32_t port = 0; + const auto* first = digits.data(); + const auto* last = digits.data() + digits.size(); + if (auto [ptr, ec] = std::from_chars(first, last, port); + ec != std::errc{} || ptr != last) { + return util::unexpected(parse_error{ + L.string(), "invalid port number", port_tok}); + } + u.port = port; + } + + // optional path (begins with '/'). + if (L == lex::tok::slash) { + auto path = util::parse_string(L, "path", is_path_tok); + u.path = std::move(*path); // starts with '/', so never empty + } + + // optional query. + if (L == lex::tok::question) { + L.next(); // consume '?' + if (is_query_tok(L.current_kind())) { + auto query = util::parse_string(L, "query", is_query_tok); + u.query = std::move(*query); + } + } + + // optional fragment. + if (L == lex::tok::hash) { + L.next(); // consume '#' + if (is_fragment_tok(L.current_kind())) { + auto fragment = util::parse_string(L, "fragment", is_fragment_tok); + u.fragment = std::move(*fragment); + } + } + + if (auto e = expect_end(L, "url"); !e) { + return util::unexpected(e.error()); + } + return u; +} + +util::expected +parse_bearer_challenge(std::string_view text) { + const auto full = util::trim(text); + lex::lexer L(full); + + // scheme: a case-insensitive "Bearer" token. "Bearerish" lexes as a single + // symbol and so is rejected here; "Basic" likewise. + const auto scheme_tok = L.peek(); + if (L != lex::tok::symbol || + util::to_lower(scheme_tok.spelling) != "bearer") { + return util::unexpected(parse_error{ + full, "expected a 'Bearer' authentication scheme", scheme_tok}); + } + L.next(); // consume the scheme + if (L != lex::tok::whitespace && + L != lex::tok::end) { + const auto t = L.peek(); + return util::unexpected(parse_error{ + full, "expected whitespace after the 'Bearer' scheme", t}); + } + if (L == lex::tok::whitespace) { + L.next(); + } + + bearer_challenge challenge; + while (L != lex::tok::end) { + // parameter name. + const auto key_tok = L.peek(); + if (L != lex::tok::symbol) { + return util::unexpected(parse_error{ + full, "expected a parameter name", key_tok}); + } + const std::string key = util::to_lower(L.next().spelling); + + if (L == lex::tok::whitespace) { + L.next(); + } + if (L != lex::tok::equals) { + const auto t = L.peek(); + return util::unexpected(parse_error{ + full, "expected '=' after the parameter name", t}); + } + L.next(); // consume '=' + if (L == lex::tok::whitespace) { + L.next(); + } + + // parameter value: a quoted string (whose content may contain any byte, + // so it is read raw rather than tokenised) or a bare token run. + std::string value; + if (L == lex::tok::dquote) { + const unsigned q = L.peek().loc; // position of the opening '"' + std::size_t i = static_cast(q) + 1; + std::string buf; + bool closed = false; + while (i < full.size()) { + const char c = full[i]; + if (c == '\\' && i + 1 < full.size()) { + buf.push_back(full[i + 1]); + i += 2; + continue; + } + if (c == '"') { + closed = true; + break; + } + buf.push_back(c); + ++i; + } + if (!closed) { + return util::unexpected(parse_error{ + full, "unterminated quoted parameter value", L.peek()}); + } + L.seek(static_cast(i + 1)); // skip past the closing '"' + value = std::move(buf); + } else { + auto raw = util::parse_string(L, "value", is_unquoted_value_tok); + if (!raw) { + return util::unexpected(raw.error()); + } + value = std::string{util::trim(*raw)}; + } + + if (key == "realm") { + challenge.realm = std::move(value); + } else if (key == "service") { + challenge.service = std::move(value); + } else if (key == "scope") { + auto scopes = parse_scopes(value); + challenge.scopes.insert(challenge.scopes.end(), scopes.begin(), + scopes.end()); + } + // unknown parameters are ignored. + + // separator: a comma between parameters, or end. + if (L == lex::tok::whitespace) { + L.next(); + } + if (L == lex::tok::comma) { + L.next(); + if (L == lex::tok::whitespace) { + L.next(); + } + } else if (L != lex::tok::end) { + const auto t = L.peek(); + return util::unexpected(parse_error{ + full, "expected ',' between parameters", t}); + } + } + + if (challenge.realm.empty()) { + return util::unexpected(parse_error{ + full, "the Bearer challenge is missing the required 'realm'", + scheme_tok}); + } + return challenge; +} + +std::vector parse_scopes(std::string_view value) { + std::vector out; + lex::lexer L(value); + std::string current; + auto flush = [&] { + if (!current.empty()) { + out.push_back(std::move(current)); + current.clear(); + } + }; + while (L != lex::tok::end) { + if (L == lex::tok::whitespace) { + L.next(); + flush(); + } else { + current += L.next().spelling; + } + } + flush(); + return out; +} + +} // namespace oci diff --git a/src/oci/parse.h b/src/oci/parse.h new file mode 100644 index 00000000..ef64a921 --- /dev/null +++ b/src/oci/parse.h @@ -0,0 +1,63 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +// Lexer-based parsers for the string types used by the OCI client. These mirror +// the interface style of src/uenv/parse.* and share the util parsing scaffolding +// (util::parse_error, the PARSE macro, util::parse_string) so that src/oci stays +// self-contained on src/util. JSON documents are still parsed with nlohmann; the +// parsers here handle the character-level string types only. +namespace oci { + +// parse an OCI content digest ":". rejects an unknown algorithm +// and a value whose length does not match the algorithm or that is not lowercase +// hex. +util::expected parse_digest(std::string_view text); + +// parse a manifest reference: a valid ":" becomes a digest reference, +// otherwise the text is validated against the OCI tag grammar and becomes a tag. +util::expected +parse_reference(std::string_view text); + +// A URL parsed into its RFC-3986 components. `host` retains the surrounding +// brackets for an IPv6 literal (e.g. "[::1]"). Percent-encoded octets are passed +// through verbatim (not decoded). +struct url { + std::string scheme; // e.g. "https" (may be empty) + std::string userinfo; // e.g. "user:pass" (may be empty) + std::string host; // e.g. "jfrog.svc.cscs.ch" or "[::1]" + std::optional port; + std::string path; // includes the leading '/' (may be empty) + std::string query; // after '?' (may be empty) + std::string fragment; // after '#' (may be empty) + + // re-render the URL from its components. + std::string string() const; + + friend bool operator==(const url&, const url&) = default; +}; + +// parse a URL "[scheme://][userinfo@]host[:port][/path][?query][#fragment]". a +// bare "host/prefix" (no scheme) is accepted, as registry configs are written. +util::expected parse_url(std::string_view text); + +// parse a "WWW-Authenticate: Bearer ..." challenge. the scheme match is +// case-insensitive; `realm` is required. +util::expected +parse_bearer_challenge(std::string_view text); + +// split a scope value on whitespace (the OCI spec permits a single scope +// parameter carrying a space-separated list). never fails. +std::vector parse_scopes(std::string_view value); + +} // namespace oci diff --git a/src/oci/pull.cpp b/src/oci/pull.cpp index af96406e..24df5b2f 100644 --- a/src/oci/pull.cpp +++ b/src/oci/pull.cpp @@ -3,22 +3,18 @@ #include #include -#include #include +#include +#include +#include +#include #include #include #include -#include -#include namespace oci { -// defined in client.cpp (part of oci::impl, kept out of the public interface). -namespace impl { -std::string digest_string(const util::sha256_digest& d); -} - namespace fs = std::filesystem; namespace { @@ -43,8 +39,7 @@ util::expected extract_targz(const std::string& data, } } - auto proc = util::run( - {"tar", "-xzf", tmp.string(), "-C", dest.string()}); + auto proc = util::run({"tar", "-xzf", tmp.string(), "-C", dest.string()}); if (!proc) { fs::remove(tmp); return util::unexpected{ @@ -59,52 +54,43 @@ util::expected extract_targz(const std::string& data, return {}; } +util::expected ensure_dir(const fs::path& store) { + if (fs::exists(store)) { + return {}; + } + std::error_code ec; + fs::create_directories(store, ec); + if (ec) { + return util::unexpected{ + fmt::format("unable to create {}: {}", store.string(), ec.message())}; + } + return {}; +} + } // namespace util::expected -pull_squashfs(client& c, const manifest_response& manifest, - const fs::path& store, progress_fn progress, - std::function should_abort) { - auto j = nlohmann::json::parse(manifest.body, nullptr, - /*allow_exceptions=*/false); - if (j.is_discarded() || !j.contains("layers")) { - return util::unexpected{"image manifest has no layers"}; - } - +pull_squashfs(client& c, const manifest& image, const fs::path& store, + progress_fn progress, std::function should_abort) { // pick the squashfs layer: prefer the one titled "store.squashfs", else the // sole layer. - std::string digest; - for (const auto& layer : j["layers"]) { - auto ann = layer.find("annotations"); - if (ann != layer.end()) { - auto title = ann->find("org.opencontainers.image.title"); - if (title != ann->end() && title->is_string() && - title->get() == squashfs_title) { - digest = layer.value("digest", ""); - break; - } - } + const manifest_layer* layer = image.find_layer_by_title(squashfs_title); + if (!layer && image.layers.size() == 1) { + layer = &image.layers[0]; } - if (digest.empty() && j["layers"].size() == 1) { - digest = j["layers"][0].value("digest", ""); - } - if (digest.empty()) { + if (!layer) { return util::unexpected{ "could not find the store.squashfs layer in the manifest"}; } + const digest& want = layer->digest; - if (!fs::exists(store)) { - std::error_code ec; - fs::create_directories(store, ec); - if (ec) { - return util::unexpected{fmt::format("unable to create {}: {}", - store.string(), ec.message())}; - } + if (auto ok = ensure_dir(store); !ok) { + return util::unexpected{ok.error()}; } const fs::path dest = store / "store.squashfs"; - spdlog::debug("oci::pull_squashfs {} -> {}", digest, dest.string()); - if (auto ok = c.get_blob_to_file(digest, dest, std::move(progress), + spdlog::debug("oci::pull_squashfs {} -> {}", want.string(), dest.string()); + if (auto ok = c.get_blob_to_file(want, dest, std::move(progress), std::move(should_abort)); !ok) { return util::unexpected{ok.error()}; @@ -115,79 +101,64 @@ pull_squashfs(client& c, const manifest_response& manifest, if (!actual) { return util::unexpected{actual.error()}; } - auto actual_digest = impl::digest_string(*actual); - if (actual_digest != digest) { + auto got = digest::from_sha256(*actual); + if (got != want) { std::error_code ec; fs::remove(dest, ec); return util::unexpected{fmt::format( - "downloaded squashfs digest mismatch: expected {}, got {}", digest, - actual_digest)}; + "downloaded squashfs digest mismatch: expected {}, got {}", + want.string(), got.string())}; } - spdlog::debug("oci::pull_squashfs verified {}", actual_digest); + spdlog::debug("oci::pull_squashfs verified {}", got.string()); return {}; } util::expected -pull_meta(client& c, const std::string& manifest_digest, const fs::path& store) { +pull_meta(client& c, const digest& manifest_digest, const fs::path& store) { auto refs = c.referrers(manifest_digest); if (!refs) { return util::unexpected{refs.error()}; } // find the uenv/meta referrer. - std::string meta_manifest_digest; + const descriptor* meta = nullptr; for (const auto& r : *refs) { - if (r.artifact_type == "uenv/meta") { - meta_manifest_digest = r.digest; + if (r.artifact_type == artifact_type_meta) { + meta = &r; break; } } - if (meta_manifest_digest.empty()) { + if (!meta) { return false; // no attached meta } - auto meta_manifest = c.get_manifest(meta_manifest_digest); + auto meta_manifest = c.get_manifest(reference::digest(meta->digest)); if (!meta_manifest) { return util::unexpected{meta_manifest.error()}; } - auto j = nlohmann::json::parse(meta_manifest->body, nullptr, - /*allow_exceptions=*/false); - if (j.is_discarded() || !j.contains("layers") || j["layers"].empty()) { - return util::unexpected{"meta manifest has no layers"}; + auto parsed = parse_manifest(meta_manifest->body); + if (!parsed) { + return util::unexpected{parsed.error()}; } - // the meta layer is the gzipped tar marked for unpacking. - std::string digest; - for (const auto& layer : j["layers"]) { - auto ann = layer.find("annotations"); - if (ann != layer.end()) { - auto unpack = ann->find("io.deis.oras.content.unpack"); - if (unpack != ann->end() && unpack->is_string() && - unpack->get() == "true") { - digest = layer.value("digest", ""); - break; - } - } + // the meta layer is the gzipped tar marked for unpacking (else the sole one). + const manifest_layer* layer = parsed->find_unpack_layer(); + if (!layer && !parsed->layers.empty()) { + layer = &parsed->layers[0]; } - if (digest.empty()) { - digest = j["layers"][0].value("digest", ""); + if (!layer) { + return util::unexpected{"meta manifest has no layers"}; } - spdlog::debug("oci::pull_meta layer {}", digest); - auto blob = c.get_blob(digest); + spdlog::debug("oci::pull_meta layer {}", layer->digest.string()); + auto blob = c.get_blob(layer->digest); if (!blob) { return util::unexpected{blob.error()}; } - if (!fs::exists(store)) { - std::error_code ec; - fs::create_directories(store, ec); - if (ec) { - return util::unexpected{fmt::format("unable to create {}: {}", - store.string(), ec.message())}; - } + if (auto ok = ensure_dir(store); !ok) { + return util::unexpected{ok.error()}; } - if (auto ok = extract_targz(*blob, store); !ok) { return util::unexpected{ok.error()}; } diff --git a/src/oci/pull.h b/src/oci/pull.h index 500bedf8..be2c192d 100644 --- a/src/oci/pull.h +++ b/src/oci/pull.h @@ -6,6 +6,8 @@ #include #include +#include +#include #include namespace oci { @@ -13,13 +15,13 @@ namespace oci { // progress callback for the squashfs download: (bytes_downloaded, bytes_total). using progress_fn = std::function; -// Download the squashfs layer of `manifest` into /store.squashfs and +// Download the squashfs layer of `image` into /store.squashfs and // self-verify: the file is re-digested and checked against the layer digest, so -// a truncated or corrupt download is caught locally. `manifest` is the image -// manifest already fetched via client::get_manifest. An optional abort -// predicate, polled during the download, cancels it (for Ctrl-C handling). +// a truncated or corrupt download is caught locally. `image` is the parsed image +// manifest. An optional abort predicate, polled during the download, cancels it +// (for Ctrl-C handling). util::expected -pull_squashfs(client& c, const manifest_response& manifest, +pull_squashfs(client& c, const manifest& image, const std::filesystem::path& store, progress_fn progress = {}, std::function should_abort = {}); @@ -27,7 +29,7 @@ pull_squashfs(client& c, const manifest_response& manifest, // reproducing the `meta/` directory. Returns true if meta was found and // pulled, false if the image has no attached meta. util::expected -pull_meta(client& c, const std::string& manifest_digest, +pull_meta(client& c, const digest& manifest_digest, const std::filesystem::path& store); } // namespace oci diff --git a/src/oci/push.cpp b/src/oci/push.cpp new file mode 100644 index 00000000..6ff42cda --- /dev/null +++ b/src/oci/push.cpp @@ -0,0 +1,431 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace oci { + +// defined in client.cpp (part of oci::impl, kept out of the public interface). +// parses the "manifests" array of an OCI image index into descriptors. +namespace impl { +std::optional> parse_referrers(std::string_view body); +} + +namespace fs = std::filesystem; + +namespace { + +// current time as an RFC3339 UTC timestamp (e.g. 2024-08-23T16:00:40Z), the +// format oras writes into org.opencontainers.image.created. +std::string rfc3339_now() { + std::time_t t = std::time(nullptr); + std::tm tm{}; + gmtime_r(&t, &tm); + char buf[32]; + std::strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%SZ", &tm); + return buf; +} + +util::expected digest_of_file(const fs::path& p) { + auto d = util::sha256_file(p); + if (!d) { + return util::unexpected{d.error()}; + } + return digest::from_sha256(*d); +} + +digest digest_of_string(std::string_view s) { + return digest::from_sha256(util::sha256_string(s)); +} + +// upload the canonical empty config blob (idempotent). +util::expected put_empty_config(client& c) { + return c.put_blob_bytes(empty_config_descriptor().digest, + std::string{empty_config_body}); +} + +// A packaged payload ready to become a manifest layer: the blob on disk plus the +// layer descriptor, and a scratch directory to clean up after upload. +struct packaged_layer { + fs::path blob; + manifest_layer layer; + std::optional scratch = std::nullopt; +}; + +// Pack a directory into a deterministic gzipped tar, mirroring what +// `oras attach` produces for a directory payload: a single tar+gzip layer +// annotated for unpacking, titled with the directory name, and carrying the +// uncompressed-tar digest. Deterministic tar/gzip flags keep the layer digest +// stable across runs. +util::expected +package_directory(const fs::path& dir) { + const auto parent = dir.parent_path(); + const auto name = dir.filename().string(); + + auto work = util::make_temp_dir(); + const auto tar_path = work / "payload.tar"; + + auto tar = util::run({"tar", "--sort=name", "--format=posix", "--mtime=@0", + "--owner=0", "--group=0", "--numeric-owner", "-cf", + tar_path.string(), "-C", parent.string(), name}); + if (!tar) { + return util::unexpected{fmt::format( + "unable to run tar to pack {}: {}", dir.string(), tar.error())}; + } + if (auto rc = tar->wait(); rc != 0) { + return util::unexpected{ + fmt::format("tar failed to pack {} (exit {})", dir.string(), rc)}; + } + + // digest of the uncompressed tar (the io.deis.oras.content.digest value). + auto tar_digest = digest_of_file(tar_path); + if (!tar_digest) { + return util::unexpected{tar_digest.error()}; + } + + // gzip in place; -n omits the filename/timestamp for a reproducible result. + auto gz = util::run({"gzip", "-n", tar_path.string()}); + if (!gz) { + return util::unexpected{fmt::format("unable to run gzip: {}", gz.error())}; + } + if (auto rc = gz->wait(); rc != 0) { + return util::unexpected{fmt::format("gzip failed (exit {})", rc)}; + } + const auto gz_path = fs::path{tar_path.string() + ".gz"}; + + auto layer_digest = digest_of_file(gz_path); + if (!layer_digest) { + return util::unexpected{layer_digest.error()}; + } + std::error_code ec; + auto size = fs::file_size(gz_path, ec); + if (ec) { + return util::unexpected{fmt::format("unable to stat {}: {}", + gz_path.string(), ec.message())}; + } + + return packaged_layer{ + .blob = gz_path, + .layer = manifest_layer{.media_type = std::string{media_type_layer_targz}, + .digest = *layer_digest, + .size = size, + .annotations = + {{std::string{annotation_content_digest}, + tar_digest->string()}, + {std::string{annotation_unpack}, "true"}, + {std::string{annotation_title}, name}}}, + .scratch = work}; +} + +// Package a single file as a raw blob layer (no unpack), titled with its +// filename, with a media type guessed from the extension. +util::expected package_file(const fs::path& file) { + auto layer_digest = digest_of_file(file); + if (!layer_digest) { + return util::unexpected{layer_digest.error()}; + } + std::error_code ec; + auto size = fs::file_size(file, ec); + if (ec) { + return util::unexpected{fmt::format("unable to stat {}: {}", + file.string(), ec.message())}; + } + + std::string media_type{media_type_octet_stream}; + if (file.extension() == ".json") { + media_type = "application/json"; + } + + return packaged_layer{ + .blob = file, + .layer = manifest_layer{ + .media_type = media_type, + .digest = *layer_digest, + .size = size, + .annotations = {{std::string{annotation_title}, + file.filename().string()}}}}; +} + +// the blob digests a manifest references (config + layers). +std::vector blob_digests(const manifest& m) { + std::vector out; + out.push_back(m.config.digest); + for (const auto& l : m.layers) { + out.push_back(l.digest); + } + return out; +} + +// copy one blob from `from_repo` into `dst`'s repository, preferring a +// server-side cross-repo mount and falling back to streaming it through a local +// temp file when the registry declines the mount. +util::expected +copy_blob(client& src, client& dst, const std::string& from_repo, + const digest& d) { + auto mounted = dst.mount_blob(d, from_repo); + if (!mounted) { + return util::unexpected{mounted.error()}; + } + if (*mounted) { + spdlog::trace("copy_blob mounted {}", d.string()); + return {}; + } + + // fallback: pull the blob to a scratch file, then push it. + spdlog::debug("copy_blob streaming {} (mount declined)", d.string()); + auto work = util::make_temp_dir(); + const auto tmp = work / "blob"; + if (auto ok = src.get_blob_to_file(d, tmp); !ok) { + return util::unexpected{ok.error()}; + } + if (auto ok = dst.put_blob(d, tmp); !ok) { + return util::unexpected{ok.error()}; + } + std::error_code ec; + fs::remove(tmp, ec); + return {}; +} + +// After a referrer manifest is pushed, keep the referrers-tag index +// (-) up to date, for registries that do not auto-index the +// Referrers API. Best-effort: failures are logged, not fatal. +void maintain_referrers_tag(client& c, const digest& subject, + const descriptor& referrer) { + const auto tag = reference::tag(subject.algorithm() + "-" + subject.hex()); + + // read the existing index (an OCI image index, i.e. a manifests[] list) so + // we merge rather than clobber prior referrers. A 404/parse-miss is treated + // as an empty index. + std::vector manifests; + if (auto existing = c.get_manifest(tag)) { + if (auto refs = impl::parse_referrers(existing->body)) { + manifests = std::move(*refs); + } + } + for (const auto& d : manifests) { + if (d.digest == referrer.digest) { + return; // already indexed + } + } + manifests.push_back(referrer); + + auto body = serialize_index(manifests); + if (auto put = c.put_manifest(tag, body, media_type_index); !put) { + spdlog::warn("unable to update referrers tag {}: {}", tag.string(), + put.error()); + } +} + +} // namespace + +util::expected +push_squashfs(client& c, const fs::path& squashfs, const reference& ref) { + auto layer_digest = digest_of_file(squashfs); + if (!layer_digest) { + return util::unexpected{layer_digest.error()}; + } + std::error_code ec; + auto size = fs::file_size(squashfs, ec); + if (ec) { + return util::unexpected{fmt::format("unable to stat {}: {}", + squashfs.string(), ec.message())}; + } + + spdlog::debug("oci::push_squashfs {} ({} bytes) -> {}", + layer_digest->string(), size, ref.string()); + + // upload the squashfs blob (streamed) and the empty config. + if (auto ok = c.put_blob(*layer_digest, squashfs); !ok) { + return util::unexpected{ok.error()}; + } + if (auto ok = put_empty_config(c); !ok) { + return util::unexpected{ok.error()}; + } + + manifest m; + m.artifact_type = std::string{artifact_type_squashfs}; + m.annotations[std::string{annotation_created}] = rfc3339_now(); + m.layers.push_back(manifest_layer{ + .media_type = std::string{media_type_layer_tar}, + .digest = *layer_digest, + .size = size, + .annotations = {{std::string{annotation_title}, "store.squashfs"}}}); + + auto body = serialize_manifest(m); + if (auto ok = c.put_manifest(ref, body); !ok) { + return util::unexpected{ok.error()}; + } + + // the canonical id is the digest of the manifest bytes we just PUT. + return digest_of_string(body); +} + +util::expected +attach(client& c, const reference& subject, std::string_view artifact_type, + const fs::path& payload) { + if (!fs::exists(payload)) { + return util::unexpected{ + fmt::format("payload {} does not exist", payload.string())}; + } + + // resolve the subject (the image we annotate). + auto subj = c.get_manifest(subject); + if (!subj) { + return util::unexpected{fmt::format("unable to resolve subject {}: {}", + subject.string(), subj.error())}; + } + descriptor subject_desc{ + .media_type = subj->media_type.empty() ? std::string{media_type_manifest} + : subj->media_type, + .digest = subj->digest ? *subj->digest : digest_of_string(subj->body), + .size = subj->body.size()}; + + // package the payload. + util::expected packaged = + fs::is_directory(payload) ? package_directory(payload) + : package_file(payload); + if (!packaged) { + return util::unexpected{packaged.error()}; + } + auto cleanup = [&packaged] { + if (packaged->scratch) { + std::error_code ec; + fs::remove_all(*packaged->scratch, ec); + } + }; + + // upload the layer blob and empty config. + if (auto ok = c.put_blob(packaged->layer.digest, packaged->blob); !ok) { + cleanup(); + return util::unexpected{ok.error()}; + } + cleanup(); + if (auto ok = put_empty_config(c); !ok) { + return util::unexpected{ok.error()}; + } + + // build + push the referrer manifest. + manifest m; + m.artifact_type = std::string{artifact_type}; + m.annotations[std::string{annotation_created}] = rfc3339_now(); + m.subject = subject_desc; + m.layers.push_back(packaged->layer); + + auto body = serialize_manifest(m); + const auto manifest_digest = digest_of_string(body); + if (auto ok = c.put_manifest(reference::digest(manifest_digest), body); + !ok) { + return util::unexpected{ok.error()}; + } + + descriptor referrer{.media_type = std::string{media_type_manifest}, + .digest = manifest_digest, + .size = body.size(), + .artifact_type = std::string{artifact_type}}; + + // ensure the referrer is discoverable even on registries that do not + // auto-index the Referrers API. + maintain_referrers_tag(c, subject_desc.digest, referrer); + + return referrer; +} + +util::expected +copy_image(const std::string& registry_base, const std::string& src_repo, + const std::string& dst_repo, const digest& src_manifest, + const std::string& dst_tag, std::optional creds) { + auto src = client::create(registry_base, src_repo, creds); + if (!src) { + return util::unexpected{src.error()}; + } + auto dst = client::create(registry_base, dst_repo, creds); + if (!dst) { + return util::unexpected{dst.error()}; + } + // dst tokens need pull scope on the source repo for cross-repo mounts. + dst->add_pull_scope(src_repo); + + // fetch the image manifest and move its blobs. + auto mr = src->get_manifest(reference::digest(src_manifest)); + if (!mr) { + return util::unexpected{fmt::format("unable to fetch source manifest {}: {}", + src_manifest.string(), mr.error())}; + } + const digest manifest_digest = mr->digest.value_or(src_manifest); + const std::string_view manifest_media = + mr->media_type.empty() ? media_type_manifest + : std::string_view{mr->media_type}; + + auto parsed = parse_manifest(mr->body); + if (!parsed) { + return util::unexpected{parsed.error()}; + } + for (const auto& d : blob_digests(*parsed)) { + if (auto ok = copy_blob(*src, *dst, src_repo, d); !ok) { + return util::unexpected{ok.error()}; + } + } + // PUT the image manifest under the destination tag. Content is byte-for-byte + // identical, so the digest (identity) is preserved. + if (auto ok = dst->put_manifest(reference::tag(dst_tag), mr->body, + manifest_media); + !ok) { + return util::unexpected{ok.error()}; + } + + // copy referrers (the --recursive part): meta and any other attachments. + auto refs = src->referrers(manifest_digest); + if (!refs) { + // a registry without the Referrers API just has nothing to recurse into. + spdlog::debug("copy_image: no referrers for {} ({})", + manifest_digest.string(), refs.error()); + return {}; + } + for (const auto& r : *refs) { + auto rm = src->get_manifest(reference::digest(r.digest)); + if (!rm) { + return util::unexpected{fmt::format( + "unable to fetch referrer {}: {}", r.digest.string(), + rm.error())}; + } + auto rparsed = parse_manifest(rm->body); + if (!rparsed) { + return util::unexpected{rparsed.error()}; + } + for (const auto& d : blob_digests(*rparsed)) { + if (auto ok = copy_blob(*src, *dst, src_repo, d); !ok) { + return util::unexpected{ok.error()}; + } + } + const std::string_view rmedia = + rm->media_type.empty() ? media_type_manifest + : std::string_view{rm->media_type}; + // the referrer's subject digest is the image manifest digest, which is + // unchanged by the copy, so the manifest is valid verbatim in dst. + if (auto ok = dst->put_manifest(reference::digest(r.digest), rm->body, + rmedia); + !ok) { + return util::unexpected{ok.error()}; + } + } + + return {}; +} + +} // namespace oci diff --git a/src/oci/push.h b/src/oci/push.h new file mode 100644 index 00000000..80104e12 --- /dev/null +++ b/src/oci/push.h @@ -0,0 +1,45 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace oci { + +// Push a squashfs image as an OCI artifact (the native replacement for +// `oras push --artifact-type application/x-squashfs`). Uploads the squashfs +// blob and the empty config, builds an oras-compatible image manifest with the +// layer titled "store.squashfs", and PUTs it under `ref` (a tag). Returns the +// manifest digest — the canonical image id. +util::expected +push_squashfs(client& c, const std::filesystem::path& squashfs, + const reference& ref); + +// Attach typed side-data to an existing image as an OCI referrer (the native +// replacement for `oras attach`). `subject` identifies the target image (tag or +// digest); `artifact_type` is e.g. "uenv/meta"; `payload` is a file or a +// directory (a directory is packed as a deterministic gzipped tar flagged for +// unpacking, matching today's uenv/meta layout). Returns the descriptor of the +// pushed referrer manifest. +util::expected +attach(client& c, const reference& subject, std::string_view artifact_type, + const std::filesystem::path& payload); + +// Copy an image (and its referrers) from `src_repo` to `dst_repo` within the +// same registry (the native replacement for `oras cp --recursive`). Blobs are +// moved by cross-repo mount where the registry allows it, else streamed through +// local disk. The destination image is tagged `dst_tag`. Manifests are copied +// byte-for-byte, so the image digest (identity) is preserved. +util::expected +copy_image(const std::string& registry_base, const std::string& src_repo, + const std::string& dst_repo, const digest& src_manifest, + const std::string& dst_tag, std::optional creds); + +} // namespace oci diff --git a/src/oci/reference.cpp b/src/oci/reference.cpp new file mode 100644 index 00000000..16d3f242 --- /dev/null +++ b/src/oci/reference.cpp @@ -0,0 +1,26 @@ +#include +#include + +#include +#include + +namespace oci { + +reference reference::tag(std::string tag) { + return reference{std::move(tag)}; +} + +reference reference::digest(oci::digest d) { + return reference{std::move(d)}; +} + +util::expected +reference::parse(std::string_view text) { + return oci::parse_reference(text); +} + +std::string reference::string() const { + return digest_ ? digest_->string() : tag_; +} + +} // namespace oci diff --git a/src/oci/reference.h b/src/oci/reference.h new file mode 100644 index 00000000..4c0caaf2 --- /dev/null +++ b/src/oci/reference.h @@ -0,0 +1,54 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include + +namespace oci { + +// A manifest reference: either a mutable tag or an immutable content digest. +// Both occupy the same slot in the registry path (/manifests/), but +// they are different concepts — a tag can move, a digest is content-addressed +// and self-verifying — so they are kept distinct at the type level. +class reference { + public: + // build a reference from a tag. the tag is trusted (use parse() to validate + // untrusted input). + static reference tag(std::string tag); + // build a reference from a content digest. + static reference digest(oci::digest d); + // parse text: a valid ":" becomes a digest reference, otherwise + // it is validated against the OCI tag grammar and becomes a tag reference. a + // thin forwarder to oci::parse_reference. + static util::expected + parse(std::string_view text); + + bool is_digest() const { + return digest_.has_value(); + } + bool is_tag() const { + return !digest_.has_value(); + } + // the digest, if this reference is a digest. + const std::optional& as_digest() const { + return digest_; + } + // the text that goes into the registry path (the tag, or ":"). + std::string string() const; + + friend bool operator==(const reference&, const reference&) = default; + + private: + explicit reference(std::string tag) : tag_(std::move(tag)) { + } + explicit reference(oci::digest d) : digest_(std::move(d)) { + } + std::string tag_; + std::optional digest_; +}; + +} // namespace oci diff --git a/src/uenv/parse.cpp b/src/uenv/parse.cpp index dd1d7d4c..00c09148 100644 --- a/src/uenv/parse.cpp +++ b/src/uenv/parse.cpp @@ -14,41 +14,9 @@ namespace uenv { -std::string parse_error::message() const { - return fmt::format("{}\n {}\n {}{}", detail, input, std::string(loc, ' '), - std::string(std::max(1u, width), '^')); -} - -// some pre-processor gubbins that generates code to attempt -// parsing a value using a parse_x method. unwraps and -// forwards the error if there was an error. -// much ergonomics! -#define PARSE(L, TYPE, X) \ - { \ - if (auto rval__ = parse_##TYPE(L)) \ - X = *rval__; \ - else \ - return util::unexpected(std::move(rval__.error())); \ - } - -template -util::expected -parse_string(lex::lexer& L, std::string_view type, Test&& test) { - std::string result; - while (test(L.current_kind())) { - const auto t = L.next(); - result += t.spelling; - } - - // if result is empty, nothing was parsed - if (result.empty()) { - const auto t = L.peek(); - return util::unexpected(parse_error{ - L.string(), fmt::format("unexpected '{}'", type, t.spelling), t}); - } - - return result; -} +// the PARSE macro and parse_string are defined in ; bring the +// latter into scope so the unqualified calls below resolve. +using util::parse_string; // tokens that can appear in names // names are used for uenv names, versions, tags @@ -75,7 +43,7 @@ util::expected parse_name(lex::lexer& L) { template util::expected parse_int(lex::lexer& L) { const auto t = L.peek(); - if (t.kind != lex::tok::integer) { + if (t != lex::tok::integer) { return util::unexpected(parse_error{ L.string(), fmt::format("{} is not an integer", t.spelling), t}); } @@ -142,7 +110,7 @@ util::expected parse_path(const std::string& in) { return result; } - if (const auto t = L.peek(); t.kind != lex::tok::end) { + if (const auto t = L.peek(); t != lex::tok::end) { return util::unexpected(parse_error{ L.string(), fmt::format("unexpected symbol '{}'", t.spelling), t}); } @@ -253,7 +221,7 @@ parse_uenv_label(const std::string& in) { return result; } - if (const auto t = L.peek(); t.kind != lex::tok::end) { + if (const auto t = L.peek(); t != lex::tok::end) { return util::unexpected(parse_error{ L.string(), fmt::format("unexpected symbol '{}'", t.spelling), t}); } @@ -273,8 +241,8 @@ parse_uenv_nslabel(const std::string& in) { ++i; } // check for following colons - if (L.peek(i).kind == lex::tok::colon && - L.peek(i + 1).kind == lex::tok::colon) { + if (L.peek(i) == lex::tok::colon && + L.peek(i + 1) == lex::tok::colon) { // parse the namespace name PARSE(L, name, nspace); // gobble the :: @@ -289,7 +257,7 @@ parse_uenv_nslabel(const std::string& in) { return util::unexpected(label.error()); } - if (const auto t = L.peek(); t.kind != lex::tok::end) { + if (const auto t = L.peek(); t != lex::tok::end) { return util::unexpected(parse_error{ L.string(), fmt::format("unexpected symbol '{}'", t.spelling), t}); } @@ -383,13 +351,13 @@ parse_view_args(const std::string& arg) { L.next(); // handle trailing comma elegantly - if (L.peek().kind == lex::tok::end) { + if (L.peek() == lex::tok::end) { break; } } // if parsing finished and the string has not been // consumed, and invalid token was encountered - if (const auto t = L.peek(); t.kind != lex::tok::end) { + if (const auto t = L.peek(); t != lex::tok::end) { return util::unexpected(parse_error{ L.string(), fmt::format("unexpected symbol '{}'", t.spelling), t}); } @@ -425,14 +393,14 @@ parse_env_view_description(const std::string& arg) { L.next(); // handle trailing comma elegantly - if (L.peek().kind == lex::tok::end) { + if (L.peek() == lex::tok::end) { break; } } // if parsing finished and the string has not been // consumed, and invalid token was encountered - if (const auto t = L.peek(); t.kind != lex::tok::end) { + if (const auto t = L.peek(); t != lex::tok::end) { return util::unexpected(parse_error{ L.string(), fmt::format("unexpected symbol '{}'", t.spelling), t}); } @@ -454,20 +422,20 @@ parse_uenv_args(const std::string& arg) { PARSE(L, uenv_description, desc); uenvs.push_back(std::move(desc)); - if (L.peek().kind != lex::tok::comma) { + if (L.peek() != lex::tok::comma) { break; } // eat the comma L.next(); // handle trailing comma elegantly - if (L.peek().kind == lex::tok::end) { + if (L.peek() == lex::tok::end) { break; } } // if parsing finished and the string has not been consumed, // and invalid token was encountered - if (const auto t = L.peek(); t.kind != lex::tok::end) { + if (const auto t = L.peek(); t != lex::tok::end) { return util::unexpected(parse_error{ L.string(), fmt::format("unexpected symbol '{}'", t.spelling), t}); } @@ -488,7 +456,7 @@ parse_uenv_description(const std::string& in) { // if parsing finished and the string has not been consumed, // and invalid token was encountered - if (const auto t = L.peek(); t.kind != lex::tok::end) { + if (const auto t = L.peek(); t != lex::tok::end) { return util::unexpected(parse_error{ L.string(), fmt::format("unexpected symbol '{}'", t.spelling), t}); } @@ -508,20 +476,20 @@ parse_mount_list(const std::string& arg) { PARSE(L, mount_description, mnt); mounts.push_back(std::move(mnt)); - if (L.peek().kind != lex::tok::comma) { + if (L.peek() != lex::tok::comma) { break; } // eat the comma L.next(); // handle trailing comma elegantly - if (L.peek().kind == lex::tok::end) { + if (L.peek() == lex::tok::end) { break; } } // if parsing finished and the string has not been consumed, // and invalid token was encountered - if (const auto t = L.peek(); t.kind != lex::tok::end) { + if (const auto t = L.peek(); t != lex::tok::end) { return util::unexpected(parse_error{ L.string(), fmt::format("unexpected symbol {}", t.spelling), t}); } @@ -538,37 +506,37 @@ parse_registry_entry(const std::string& in) { uenv_registry_entry r; PARSE(L, name, r.nspace); - if (L.peek().kind != lex::tok::slash) { + if (L.peek() != lex::tok::slash) { goto unexpected_symbol; } L.next(); PARSE(L, name, r.system); - if (L.peek().kind != lex::tok::slash) { + if (L.peek() != lex::tok::slash) { goto unexpected_symbol; } L.next(); PARSE(L, name, r.uarch); - if (L.peek().kind != lex::tok::slash) { + if (L.peek() != lex::tok::slash) { goto unexpected_symbol; } L.next(); PARSE(L, name, r.name); - if (L.peek().kind != lex::tok::slash) { + if (L.peek() != lex::tok::slash) { goto unexpected_symbol; } L.next(); PARSE(L, name, r.version); - if (L.peek().kind != lex::tok::slash) { + if (L.peek() != lex::tok::slash) { goto unexpected_symbol; } L.next(); PARSE(L, name, r.tag); - if (L.peek().kind != lex::tok::end) { + if (L.peek() != lex::tok::end) { goto unexpected_symbol; } L.next(); @@ -593,45 +561,45 @@ util::expected parse_uenv_date(const std::string& arg) { uenv_date date; PARSE(L, uint64, date.year); - if (L.peek().kind != lex::tok::dash) { + if (L.peek() != lex::tok::dash) { goto unexpected_symbol; } L.next(); PARSE(L, uint64, date.month); - if (L.peek().kind != lex::tok::dash) { + if (L.peek() != lex::tok::dash) { goto unexpected_symbol; } L.next(); PARSE(L, uint64, date.day); // time to finish - date was in 'yyyy-mm-dd' format - if (L.peek().kind == lex::tok::end) { + if (L.peek() == lex::tok::end) { goto validate_and_return; } // continue processing on the assumption that the date is one of // 'yyyy-mm-dd hh:mm:ss.uuuuuu+hh:mm' // 'yyyy-mm-ddThh:mm:ss.uuuuuu+hh:mm' - if (!(L.peek().kind == lex::tok::whitespace || - (L.peek().kind == lex::tok::symbol && L.peek().spelling == "T"))) { + if (!(L.peek() == lex::tok::whitespace || + (L.peek() == lex::tok::symbol && L.peek().spelling == "T"))) { goto unexpected_symbol; } // eat whitespace L.next(); PARSE(L, uint64, date.hour); - if (L.peek().kind != lex::tok::colon) { + if (L.peek() != lex::tok::colon) { goto unexpected_symbol; } L.next(); PARSE(L, uint64, date.minute); - if (L.peek().kind != lex::tok::colon) { + if (L.peek() != lex::tok::colon) { goto unexpected_symbol; } L.next(); PARSE(L, uint64, date.second); - if (!(L.peek().kind == lex::tok::end || L.peek().kind == lex::tok::dot)) { + if (!(L.peek() == lex::tok::end || L.peek() == lex::tok::dot)) { goto unexpected_symbol; } @@ -690,7 +658,7 @@ parse_cluster_name(const std::string& in) { return result; } - if (const auto t = L.peek(); t.kind != lex::tok::end) { + if (const auto t = L.peek(); t != lex::tok::end) { return util::unexpected(parse_error{ L.string(), fmt::format("unexpected symbol '{}'", t.spelling), t}); } @@ -706,7 +674,7 @@ parse_xthostname(const std::string& in) { return result; } - if (const auto t = L.peek(); t.kind != lex::tok::end) { + if (const auto t = L.peek(); t != lex::tok::end) { return util::unexpected(parse_error{ L.string(), fmt::format("unexpected symbol '{}'", t.spelling), t}); } @@ -741,7 +709,7 @@ parse_repo_name(const std::string& in) { return result; } - if (const auto t = L.peek(); t.kind != lex::tok::end) { + if (const auto t = L.peek(); t != lex::tok::end) { return util::unexpected(parse_error{ L.string(), fmt::format("unexpected symbol '{}'", t.spelling), t}); } @@ -793,7 +761,7 @@ parse_repo_label(const std::string& in) { // if parsing finished and the string has not been consumed, // and invalid token was encountered - if (const auto t = L.peek(); t.kind != lex::tok::end) { + if (const auto t = L.peek(); t != lex::tok::end) { return util::unexpected(parse_error{ L.string(), fmt::format("unexpected symbol {}", t.spelling), t}); } @@ -816,19 +784,19 @@ parse_repo_list(const std::string& in) { return util::unexpected{label.error()}; } - if (L.peek().kind != lex::tok::comma) { + if (L.peek() != lex::tok::comma) { break; } L.next(); // handle trailing comma elegantly - if (L.peek().kind == lex::tok::end) { + if (L.peek() == lex::tok::end) { break; } } // if parsing finished and the string has not been consumed, // and invalid token was encountered - if (const auto t = L.peek(); t.kind != lex::tok::end) { + if (const auto t = L.peek(); t != lex::tok::end) { return util::unexpected(parse_error{ L.string(), fmt::format("unexpected symbol {}", t.spelling), t}); } @@ -865,7 +833,7 @@ parse_config_line(const std::string& arg) { auto L = lex::lexer(line); auto skip_whitespace = [&L]() { - while (L.peek().kind == lex::tok::whitespace) { + while (L.peek() == lex::tok::whitespace) { L.next(); } }; @@ -883,7 +851,7 @@ parse_config_line(const std::string& arg) { skip_whitespace(); // check for '=' - if (L.peek().kind != lex::tok::equals) { + if (L.peek() != lex::tok::equals) { auto t = L.peek(); return util::unexpected(parse_error{ L.string(), fmt::format("expected '=', found '{}'", t.spelling), diff --git a/src/uenv/parse.h b/src/uenv/parse.h index cac05caf..5dfe6d7b 100644 --- a/src/uenv/parse.h +++ b/src/uenv/parse.h @@ -11,43 +11,15 @@ #include #include #include +#include #include namespace uenv { -/// represents an error generated when parsing a string. -/// -/// stores the input string and the location (loc) in the string where the error -/// was encountered -/// -/// there are two levels of error message: -/// - detail: a detailed low level description (e.g. "unexpected symbol ?") that -/// correlates to the loc in input -/// - description: a high level description, usually added at a higher level -/// (e.g. "invalid --uenv argument") -struct parse_error { - std::string input; - std::string description; - std::string detail; - unsigned loc; - unsigned width; - parse_error(std::string input, std::string detail, const lex::token& tok) - : input(std::move(input)), detail(std::move(detail)), loc(tok.loc), - width(tok.spelling.length()) { - } - parse_error(std::string_view input, std::string detail, - const lex::token& tok) - : input(input), detail(std::move(detail)), loc(tok.loc), - width(tok.spelling.length()) { - } - parse_error(std::string input, std::string description, std::string detail, - const lex::token& tok) - : input(std::move(input)), description(std::move(description)), - detail(std::move(detail)), loc(tok.loc), - width(tok.spelling.length()) { - } - std::string message() const; -}; +// the parsing scaffolding (parse_error, the PARSE macro, parse_string) lives in +// util so that it can be shared with other parsers (e.g. src/oci). uenv keeps +// the familiar unqualified name via this alias. +using parse_error = util::parse_error; util::expected, parse_error> parse_view_args(const std::string& arg); diff --git a/src/util/lex.cpp b/src/util/lex.cpp index 51dc9323..7971d8ee 100644 --- a/src/util/lex.cpp +++ b/src/util/lex.cpp @@ -10,6 +10,10 @@ bool operator==(const token& lhs, const token& rhs) { lhs.spelling == rhs.spelling; }; +bool operator==(const token& lhs, tok rhs) { + return lhs.kind == rhs; +} + inline bool is_alphanumeric(char c) { return std::isalnum(static_cast(c)); } @@ -62,6 +66,14 @@ class lexer_impl { return current_token; } + void seek(unsigned pos) { + if (pos > input_.size()) { + pos = input_.size(); + } + stream_ = input_.begin() + pos; + parse(); + } + tok current_kind() { return token_.kind; } @@ -146,6 +158,50 @@ class lexer_impl { character_token(tok::plus); ++stream_; return; + case '?': + character_token(tok::question); + ++stream_; + return; + case '&': + character_token(tok::amp); + ++stream_; + return; + case '~': + character_token(tok::tilde); + ++stream_; + return; + case '[': + character_token(tok::lbracket); + ++stream_; + return; + case ']': + character_token(tok::rbracket); + ++stream_; + return; + case '$': + character_token(tok::dollar); + ++stream_; + return; + case ';': + character_token(tok::semicolon); + ++stream_; + return; + case '(': + character_token(tok::lparen); + ++stream_; + return; + case ')': + character_token(tok::rparen); + ++stream_; + return; + case '\'': + character_token(tok::squote); + ++stream_; + return; + case '"': + character_token(tok::dquote); + ++stream_; + return; #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wpedantic" @@ -239,6 +295,10 @@ token lexer::peek(unsigned n) { return impl_->peek(n); } +void lexer::seek(unsigned pos) { + impl_->seek(pos); +} + tok lexer::current_kind() const { return impl_->current_kind(); } diff --git a/src/util/lex.h b/src/util/lex.h index 6bf44be7..84109f9e 100644 --- a/src/util/lex.h +++ b/src/util/lex.h @@ -13,8 +13,8 @@ enum class tok { colon, // colon ':' symbol, // consecutive alphabet or underscore // examples: prgenv, my_repo, _, _name - dash, // comma ',' - dot, // comma ',' + dash, // dash '-' + dot, // dot '.' whitespace, // spaces, tabs, etc. Contiguous white space characters are // joined together. bang, // exclamation mark '!' @@ -23,6 +23,17 @@ enum class tok { star, // '*' plus, // '+' percent, // percentage symbol '%' + question, // question mark '?' + amp, // ampersand '&' + tilde, // tilde '~' + lbracket, // left square bracket '[' + rbracket, // right square bracket ']' + dollar, // dollar sign '$' + semicolon, // semicolon ';' + lparen, // left parenthesis '(' + rparen, // right parenthesis ')' + squote, // single quote '\'' + dquote, // double quote '"' end, // end of input error, // invalid input encountered in stream }; @@ -35,6 +46,9 @@ struct token { std::string_view spelling; }; bool operator==(const token&, const token&); +// compare a token against a token kind, e.g. `t == lex::tok::slash`. C++20 +// synthesises `!=` and the reversed `tok == token` form from this. +bool operator==(const token&, tok); // std::ostream &operator<<(std::ostream &, const token &); @@ -48,6 +62,11 @@ class lexer { token next(); token peek(unsigned n = 0); + // reposition the lexer to absolute offset pos in the input and re-lex the + // token starting there. used to skip over content that is read raw rather + // than tokenised (e.g. quoted values in a WWW-Authenticate header). + void seek(unsigned pos); + // a convenience helper for checking the kind of the current token tok current_kind() const; @@ -107,6 +126,28 @@ template <> class fmt::formatter { return fmt::format_to(ctx.out(), "percent"); case lex::tok::plus: return fmt::format_to(ctx.out(), "plus"); + case lex::tok::question: + return fmt::format_to(ctx.out(), "question"); + case lex::tok::amp: + return fmt::format_to(ctx.out(), "amp"); + case lex::tok::tilde: + return fmt::format_to(ctx.out(), "tilde"); + case lex::tok::lbracket: + return fmt::format_to(ctx.out(), "lbracket"); + case lex::tok::rbracket: + return fmt::format_to(ctx.out(), "rbracket"); + case lex::tok::dollar: + return fmt::format_to(ctx.out(), "dollar"); + case lex::tok::semicolon: + return fmt::format_to(ctx.out(), "semicolon"); + case lex::tok::lparen: + return fmt::format_to(ctx.out(), "lparen"); + case lex::tok::rparen: + return fmt::format_to(ctx.out(), "rparen"); + case lex::tok::squote: + return fmt::format_to(ctx.out(), "squote"); + case lex::tok::dquote: + return fmt::format_to(ctx.out(), "dquote"); case lex::tok::end: return fmt::format_to(ctx.out(), "end"); case lex::tok::error: diff --git a/src/util/parse.cpp b/src/util/parse.cpp new file mode 100644 index 00000000..1d9e91e0 --- /dev/null +++ b/src/util/parse.cpp @@ -0,0 +1,15 @@ +#include +#include + +#include + +#include + +namespace util { + +std::string parse_error::message() const { + return fmt::format("{}\n {}\n {}{}", detail, input, std::string(loc, ' '), + std::string(std::max(1u, width), '^')); +} + +} // namespace util diff --git a/src/util/parse.h b/src/util/parse.h new file mode 100644 index 00000000..24a569f1 --- /dev/null +++ b/src/util/parse.h @@ -0,0 +1,80 @@ +#pragma once + +#include +#include + +#include + +#include +#include + +namespace util { + +/// represents an error generated when parsing a string. +/// +/// stores the input string and the location (loc) in the string where the error +/// was encountered +/// +/// there are two levels of error message: +/// - detail: a detailed low level description (e.g. "unexpected symbol ?") that +/// correlates to the loc in input +/// - description: a high level description, usually added at a higher level +/// (e.g. "invalid --uenv argument") +struct parse_error { + std::string input; + std::string description; + std::string detail; + unsigned loc; + unsigned width; + parse_error(std::string input, std::string detail, const lex::token& tok) + : input(std::move(input)), detail(std::move(detail)), loc(tok.loc), + width(tok.spelling.length()) { + } + parse_error(std::string_view input, std::string detail, + const lex::token& tok) + : input(input), detail(std::move(detail)), loc(tok.loc), + width(tok.spelling.length()) { + } + parse_error(std::string input, std::string description, std::string detail, + const lex::token& tok) + : input(std::move(input)), description(std::move(description)), + detail(std::move(detail)), loc(tok.loc), + width(tok.spelling.length()) { + } + std::string message() const; +}; + +// some pre-processor gubbins that generates code to attempt +// parsing a value using a parse_x method. unwraps and +// forwards the error if there was an error. +// much ergonomics! +#define PARSE(L, TYPE, X) \ + { \ + if (auto rval__ = parse_##TYPE(L)) \ + X = *rval__; \ + else \ + return util::unexpected(std::move(rval__.error())); \ + } + +// consume and concatenate the spelling of every token for which test() is +// true. returns an error if no token was consumed. +template +util::expected +parse_string(lex::lexer& L, std::string_view type, Test&& test) { + std::string result; + while (test(L.current_kind())) { + const auto t = L.next(); + result += t.spelling; + } + + // if result is empty, nothing was parsed + if (result.empty()) { + const auto t = L.peek(); + return util::unexpected(parse_error{ + L.string(), fmt::format("unexpected '{}'", type, t.spelling), t}); + } + + return result; +} + +} // namespace util diff --git a/subprojects/wrapdb.json b/subprojects/wrapdb.json new file mode 100644 index 00000000..8de73a48 --- /dev/null +++ b/subprojects/wrapdb.json @@ -0,0 +1 @@ +{"abseil-cpp":{"dependency_names":["absl_base","absl_container","absl_crc","absl_debugging","absl_flags","absl_hash","absl_log","absl_numeric","absl_profiling","absl_random","absl_status","absl_strings","absl_synchronization","absl_time","absl_types","absl_algorithm_container","absl_any_invocable","absl_bad_any_cast_impl","absl_bad_optional_access","absl_bad_variant_access","absl_bind_front","absl_city","absl_civil_time","absl_cleanup","absl_cord","absl_cord_internal","absl_cordz_functions","absl_cordz_handle","absl_cordz_info","absl_cordz_sample_token","absl_core_headers","absl_crc32c","absl_debugging_internal","absl_demangle_internal","absl_die_if_null","absl_examine_stack","absl_exponential_biased","absl_failure_signal_handler","absl_flags_commandlineflag","absl_flags_commandlineflag_internal","absl_flags_config","absl_flags_internal","absl_flags_marshalling","absl_flags_parse","absl_flags_private_handle_accessor","absl_flags_program_name","absl_flags_reflection","absl_flags_usage","absl_flags_usage_internal","absl_flat_hash_map","absl_flat_hash_set","absl_function_ref","absl_graphcycles_internal","absl_hashtablez_sampler","absl_inlined_vector","absl_int128","absl_leak_check","absl_log_initialize","absl_log_internal_check_op","absl_log_internal_message","absl_log_severity","absl_low_level_hash","absl_memory","absl_optional","absl_periodic_sampler","absl_random_bit_gen_ref","absl_random_distributions","absl_random_internal_distribution_test_util","absl_random_internal_platform","absl_random_internal_pool_urbg","absl_random_internal_randen","absl_random_internal_randen_hwaes","absl_random_internal_randen_hwaes_impl","absl_random_internal_randen_slow","absl_random_internal_seed_material","absl_random_random","absl_random_seed_gen_exception","absl_random_seed_sequences","absl_raw_hash_set","absl_raw_logging_internal","absl_scoped_set_env","absl_span","absl_spinlock_wait","absl_stacktrace","absl_statusor","absl_str_format","absl_str_format_internal","absl_strerror","absl_string_view","absl_strings_internal","absl_symbolize","absl_throw_delegate","absl_time_zone","absl_type_traits","absl_utility","absl_variant"],"versions":["20250814.1-1","20250127.1-2","20250127.1-1","20240722.0-4","20240722.0-3","20240722.0-2","20240722.0-1","20230802.1-2","20230802.1-1","20230802.0-3","20230802.0-2","20230802.0-1","20230125.1-5","20230125.1-4","20230125.1-3","20230125.1-2","20230125.1-1","20220623.0-2","20220623.0-1","20211102.0-3","20211102.0-2","20211102.0-1","20210324.2-4","20210324.2-3","20210324.2-2","20210324.2-1","20210324.1-4","20210324.1-3","20210324.1-2","20210324.1-1","20200923.2-1","20200225.2-3","20200225.2-2","20200225.2-1"]},"adamyaxley-obfuscate":{"dependency_names":["Obfuscate"],"versions":["1.0.0-1"]},"aklomp-base64":{"dependency_names":["base64"],"versions":["0.5.2-1"]},"apache-orc":{"dependency_names":["orc"],"versions":["2.3.0-1","2.2.2-1","2.2.1-1","2.2.0-1"]},"arduinocore-avr":{"dependency_names":["arduinocore","arduinocore-main","arduinocore-eeprom","arduinocore-hid","arduinocore-softwareserial","arduinocore-spi","arduinocore-wire"],"versions":["1.8.7-1","1.8.6-1","1.8.2-2","1.8.2-1","1.6.20-1"]},"argparse":{"dependency_names":["argparse"],"versions":["3.2-2","3.2-1","3.1-2","3.1-1","3.0-1","2.9-1","2.6-1","2.5-1","2.4-1","2.2-1"]},"argtable3":{"dependency_names":["argtable3"],"versions":["3.3.1-1","3.3.0-1","3.2.2-1"]},"argu-parser":{"dependency_names":["argu-parser"],"versions":["0.0.1-1"]},"argus":{"dependency_names":["argus"],"versions":["0.2.1-1","0.1.0-1"]},"arrow-cpp":{"dependency_names":["arrow","arrow-compute"],"versions":["21.0.0-1"]},"asio":{"dependency_names":["asio"],"versions":["1.36.0-2","1.36.0-1","1.30.2-2","1.30.2-1","1.28.1-1","1.24.0-1"]},"atomic_queue":{"dependency_names":["atomic_queue"],"versions":["1.9.2-1","1.7.1-1","1.7.0-1","1.6.9-1","1.6.7-1","1.6.6-1"]},"au":{"dependency_names":["au"],"versions":["0.5.0-1","0.4.1-1"]},"aws-c-cal":{"dependency_names":["aws-c-cal"],"versions":["0.9.10-1"]},"aws-c-common":{"dependency_names":["aws-c-common"],"versions":["0.12.4-3","0.12.4-2","0.12.4-1"]},"aws-c-compression":{"dependency_names":["aws-c-compression"],"program_names":["aws-c-common-huffman-generator"],"versions":["0.3.1-1"]},"backward-cpp":{"dependency_names":["backward-cpp","backward-cpp-interface"],"versions":["1.6-6","1.6-5","1.6-4","1.6-3","1.6-2","1.6-1"]},"bdwgc":{"dependency_names":["bdw-gc"],"versions":["8.2.10-1","8.2.8-1","8.2.2-1","8.2.0-1","7.6.8-1"]},"blueprint-compiler":{"program_names":["blueprint-compiler"],"versions":["0.18.0-2","0.18.0-1","0.16.0-1","0.10.0-1"]},"boost-mp11":{"dependency_names":["boost-mp11"],"versions":["1.88.0-1","1.83.0-1"]},"boost-pfr":{"dependency_names":["boost-pfr"],"versions":["1.88.0-1"]},"box2d":{"dependency_names":["box2d"],"versions":["2.4.1-4","2.4.1-3","2.4.1-2","2.4.1-1","2.3.1-7","2.3.1-5","2.3.1-4","2.3.1-2"]},"bshoshany-thread-pool":{"dependency_names":["bshoshany-thread-pool"],"versions":["5.1.0-1","5.0.0-1","4.1.0-1","4.0.1-1","4.0.0-1","3.5.0-1","3.4.0-1","3.3.0-1"]},"bzip2":{"dependency_names":["bzip2"],"versions":["1.0.8-2","1.0.8-1"]},"c-ares":{"dependency_names":["libcares"],"versions":["1.34.6-1","1.34.5-1","1.34.4-1","1.24.0-1","1.22.1-2","1.22.1-1","1.20.1-1"]},"c-flags":{"dependency_names":["c-flags"],"versions":["1.5.10-1","1.5.8-1","1.5.4-1"]},"cairo":{"dependency_names":["cairo","cairo-gobject","cairo-script-interpreter"],"versions":["1.18.4-2","1.18.4-1","1.18.2-1","1.18.0-1","1.17.8-1"]},"cairomm":{"dependency_names":["cairomm-1.16"],"versions":["1.18.0-1"]},"catch":{"versions":["2.2.2-1","2.2.1-2","2.2.1-1"]},"catch2":{"dependency_names":["catch2","catch2-with-main"],"versions":["3.14.0-1","3.13.0-1","3.12.0-1","3.11.0-1","3.10.0-1","3.9.1-1","3.8.1-1","3.8.0-1","3.7.1-1","3.7.0-1","3.6.0-1","3.5.4-1","3.5.3-1","3.5.2-1","3.4.0-1","3.3.2-1","3.2.0-1","3.1.0-1","2.13.8-1","2.13.7-1","2.13.3-2","2.13.3-1","2.11.3-1","2.11.1-1","2.9.0-1","2.8.0-1","2.7.2-1","2.5.0-1","2.4.1-1"]},"catchorg-clara":{"versions":["1.1.5-1"]},"centurion":{"dependency_names":["centurion"],"versions":["7.3.0-1","7.2.0-1"]},"cereal":{"dependency_names":["cereal"],"versions":["1.3.2-1","1.3.0-1","1.2.2-1"]},"cexception":{"dependency_names":["cexception"],"versions":["1.3.4-1","1.3.3-1"]},"cglm":{"dependency_names":["cglm"],"versions":["0.9.6-1","0.9.4-1","0.9.2-1"]},"check":{"dependency_names":["check"],"versions":["0.15.2-6","0.15.2-5","0.15.2-4","0.15.2-3","0.15.2-2","0.15.2-1","0.14.0-1"]},"chipmunk":{"dependency_names":["chipmunk"],"versions":["7.0.3-1","6.2.2-2","6.2.2-1"]},"cjson":{"dependency_names":["libcjson","libcjson_utils"],"versions":["1.7.19-1","1.7.18-2","1.7.18-1","1.7.17-1","1.7.16-2","1.7.16-1","1.7.15-6","1.7.15-5","1.7.15-4","1.7.15-3","1.7.15-2","1.7.15-1","1.7.14-1"]},"cli11":{"dependency_names":["CLI11"],"versions":["2.6.2-1","2.6.1-1","2.6.0-1","2.5.0-2","2.5.0-1","2.4.2-1","2.4.1-1","2.3.2-1","2.2.0-1","2.1.2-1","1.9.1-1"]},"cmark-gfm":{"dependency_names":["libcmark-gfm"],"versions":["0.29.0.gfm.13-1","0.29.0.gfm.10-1","0.29.0.gfm.6-1"]},"cmock":{"dependency_names":["cmock"],"versions":["2.6.0-1","2.5.3-1"]},"cmocka":{"dependency_names":["cmocka"],"versions":["1.1.8-1","1.1.7-6","1.1.7-5","1.1.7-4","1.1.7-3","1.1.7-2","1.1.7-1","1.1.5-5","1.1.5-4","1.1.5-3","1.1.5-2","1.1.5-1","1.1.2-1"]},"concurrentqueue":{"dependency_names":["concurrentqueue"],"versions":["1.0.4-1","1.0.3-1"]},"cpp-httplib":{"dependency_names":["cpp-httplib"],"versions":["0.48.0-1","0.47.0-1","0.46.1-1","0.46.0-1","0.45.0-1","0.43.2-1","0.43.1-1","0.42.0-1","0.41.0-1","0.40.0-1","0.39.0-1","0.38.0-1","0.37.2-1","0.37.1-1","0.37.0-1","0.36.0-1","0.35.0-1","0.34.0-1","0.33.1-1","0.32.0-1","0.31.0-1","0.30.2-1","0.30.1-1","0.30.0-1","0.29.0-1","0.28.0-1","0.27.0-1","0.26.0-1","0.25.0-1","0.24.0-1","0.23.1-1","0.22.0-1","0.21.0-1","0.20.1-1","0.20.0-1","0.19.0-1","0.18.7-1","0.18.6-1","0.18.5-1","0.18.3-1","0.18.2-1","0.18.1-1","0.18.0-1","0.17.3-1","0.17.1-1","0.16.3-1","0.15.3-1","0.13.1-1","0.11.2-1","0.8.9-2","0.8.9-1"]},"cpp-semver":{"dependency_names":["cpp-semver"],"versions":["0.3.3-1"]},"cpputest":{"dependency_names":["cpputest"],"versions":["4.0-4","4.0-3","4.0-2","4.0-1"]},"cppzmq":{"dependency_names":["cppzmq"],"versions":["4.11.0-2","4.11.0-1","4.10.0-4","4.10.0-3","4.10.0-2","4.10.0-1","4.9.0-1","4.8.1-2","4.8.1-1"]},"cpr":{"dependency_names":["cpr"],"versions":["1.14.2-1","1.14.1-1","1.12.0-1","1.11.2-1","1.11.1-1","1.11.0-1","1.10.5-1","1.10.4-1","1.9.6-2","1.9.6-1","1.9.3-1","1.9.2-2","1.9.2-1","1.7.2-1","1.5.0-1","1.3.0-1"]},"croaring":{"dependency_names":["croaring"],"versions":["4.3.11-1","1.3.0-2","1.3.0-1"]},"crow":{"dependency_names":["Crow"],"versions":["1.3.0-1"]},"csfml":{"dependency_names":["csfml"],"versions":["3.0.0-1"]},"ctre":{"dependency_names":["ctre"],"versions":["3.10.0-1","3.9.0-1"]},"curl":{"dependency_names":["libcurl"],"versions":["8.12.1-2","8.12.1-1","8.10.1-1","8.10.0-1","8.9.1-2","8.9.1-1","8.9.0-1","8.8.0-1","8.7.1-1","8.6.0-1","8.5.0-2","8.5.0-1","8.4.0-2","8.4.0-1","8.3.0-1","8.0.1-1"]},"cwalk":{"dependency_names":["cwalk"],"versions":["1.2.9-2","1.2.9-1"]},"cxxopts":{"dependency_names":["cxxopts"],"versions":["3.2.0-1","3.1.1-1","3.0.0-1","2.2.1-1","2.2.0-2","2.2.0-1"]},"debug_assert":{"dependency_names":["debug_assert"],"versions":["1.3.4-1","1.3.3-1"]},"directxmath":{"dependency_names":["directxmath"],"versions":["3.1.9-2","3.1.9-1","3.1.8-1"]},"dlfcn-win32":{"dependency_names":["dl"],"versions":["1.4.2-1","1.4.1-1","1.3.1-1"]},"docopt":{"dependency_names":["docopt"],"versions":["0.6.3-5","0.6.3-4","0.6.3-3","0.6.3-2","0.6.3-1"]},"doctest":{"dependency_names":["doctest"],"versions":["2.5.2-1","2.5.1-1","2.5.0-1","2.4.12-1","2.4.11-1","2.4.9-1","2.4.8-1"]},"dragonbox":{"dependency_names":["dragonbox"],"versions":["1.1.2-1"]},"dyninst":{"dependency_names":["dyninst"],"versions":["13.0.0-2","13.0.0-1"]},"eigen":{"dependency_names":["eigen3"],"versions":["5.0.1-1","5.0.0-1","3.4.0-2","3.4.0-1","3.3.9-1","3.3.8-1","3.3.7-3","3.3.7-2","3.3.7-1","3.3.5-1","3.3.4-1"]},"emilk-loguru":{"dependency_names":["loguru"],"versions":["2.1.0-10","2.1.0-9","2.1.0-8","2.1.0-7","2.1.0-6","2.1.0-5","2.1.0-4","2.1.0-3","2.1.0-2","2.1.0-1"]},"enet":{"dependency_names":["enet"],"versions":["1.3.18-1","1.3.17-2","1.3.17-1","1.3.13-4","1.3.13-2"]},"enlog":{"dependency_names":["enlog"],"versions":["1.8-1","1.6-1"]},"entt":{"dependency_names":["entt"],"versions":["3.11.0-3","3.11.0-2","3.11.0-1"]},"epoxy":{"dependency_names":["epoxy"],"versions":["1.5.10-2","1.5.10-1","1.5.9-1"]},"exiv2":{"dependency_names":["exiv2"],"versions":["0.28.8-1","0.28.7-1","0.28.5-1","0.28.4-1","0.28.3-1","0.28.1-1","0.28.0-1"]},"expat":{"dependency_names":["expat"],"versions":["2.8.2-1","2.7.5-1","2.7.4-1","2.7.3-1","2.7.1-1","2.6.4-1","2.6.3-2","2.6.3-1","2.6.2-1","2.6.0-1","2.5.0-4","2.5.0-3","2.5.0-2","2.5.0-1","2.4.9-1","2.4.8-2","2.4.8-1","2.2.9-4","2.2.9-3","2.2.9-2","2.2.9-1","2.2.6-1","2.2.5-5","2.2.5-4","2.2.5-3","2.2.5-1"]},"eyalroz-printf":{"dependency_names":["eyalroz-printf"],"versions":["6.2.0-1"]},"facil":{"dependency_names":["facil"],"versions":["0.7.6-5","0.7.6-4","0.7.6-3","0.7.6-2","0.7.6-1","0.7.5-1"]},"fast_float":{"dependency_names":["FastFloat"],"versions":["8.2.4-1"]},"fdk-aac":{"dependency_names":["fdk-aac"],"versions":["2.0.3-2","2.0.3-1","2.0.2-0"]},"ff-nvcodec-headers":{"dependency_names":["ffnvcodec"],"versions":["11.1.5.1-0"]},"flac":{"dependency_names":["flac"],"versions":["1.5.0-3","1.5.0-2","1.5.0-1","1.4.3-2","1.4.3-1","1.4.2-2","1.4.2-1","1.4.1-1","1.4.0-2","1.4.0-1","1.3.4-4","1.3.4-3","1.3.4-2","1.3.4-1","1.3.3-2","1.3.3-1"]},"flatbuffers":{"dependency_names":["flatbuffers"],"program_names":["flatc","flathash"],"versions":["24.3.25-1","24.3.6-1","23.3.3-1","23.1.21-1","22.11.23-1","2.0.8-1","2.0.6-2","2.0.6-1","1.11.0-1"]},"fluidsynth":{"dependency_names":["fluidsynth"],"versions":["2.3.3-2","2.3.3-1","2.3.2-1","2.3.0-1","2.2.8-1","2.2.6-1","2.2.4-3","2.2.4-2","2.2.4-1","2.2.3-2","2.2.3-1","2.2.0-1","2.1.8-1","2.1.7-1"]},"fmt":{"dependency_names":["fmt"],"versions":["12.0.0-1","11.2.0-2","11.2.0-1","11.1.4-1","11.1.1-2","11.1.1-1","11.0.2-1","11.0.1-1","10.2.0-2","10.2.0-1","10.1.1-1","9.1.0-2","9.1.0-1","9.0.0-1","8.1.1-2","8.1.1-1","8.0.1-1","7.1.3-1","7.0.1-1","6.2.0-2","6.2.0-1","6.0.0-1","5.3.0-1","5.2.1-1","5.2.0-1","4.1.0-1"]},"fontconfig":{"dependency_names":["fontconfig"],"versions":["2.16.2-1","2.16.1-1","2.16.0-1","2.15.0-1","2.14.2-1"]},"freeglut":{"dependency_names":["freeglut","glut"],"versions":["3.8.0-2","3.8.0-1","3.6.0-1","3.4.0-3","3.4.0-2","3.4.0-1"]},"freetype2":{"dependency_names":["freetype2"],"versions":["2.14.3-1","2.14.2-1","2.14.1-1","2.14.0-1","2.13.3-2","2.13.3-1","2.13.2-1","2.13.1-1","2.13.0-1","2.12.1-2","2.12.1-1","2.12.0-1","2.11.1-1","2.11.0-1","2.9.1-2","2.9.1-1"]},"fribidi":{"dependency_names":["fribidi"],"versions":["1.0.16-1","1.0.15-1","1.0.13-1"]},"frozen":{"dependency_names":["frozen"],"versions":["1.2.0-1","1.1.1-1","1.0.1-2","1.0.1-1"]},"ftxui":{"dependency_names":["ftxui-screen","ftxui-dom","ftxui-component"],"versions":["6.1.9-1","5.0.0-1","4.0.0-1","3.0.0-2","3.0.0-1","2.0.0-3","2.0.0-2","2.0.0-1"]},"fuse":{"dependency_names":["fuse"],"versions":["2.9.9-4","2.9.9-3","2.9.9-2","2.9.9-1"]},"gdbm":{"dependency_names":["gdbm"],"versions":["1.26-2","1.26-1","1.24-1","1.23-2","1.23-1","1.14.1-2","1.14.1-1"]},"gdk-pixbuf":{"dependency_names":["gdk-pixbuf-2.0"],"versions":["2.44.7-1","2.44.6-1","2.44.5-1","2.44.4-1","2.44.3-1","2.44.2-1","2.44.1-1","2.44.0-1","2.42.12-1","2.42.10-1","2.42.9-1"]},"gee":{"dependency_names":["gee-0.8"],"versions":["0.20.8-1","0.20.6-1"]},"gflags":{"dependency_names":["gflags"],"versions":["2.2.2-1"]},"giflib":{"dependency_names":["giflib"],"versions":["5.2.2-3","5.2.2-2","5.2.2-1","5.2.1-3","5.2.1-2","5.2.1-1"]},"glbinding":{"dependency_names":["glbinding","glbinding-aux"],"versions":["3.5.0-1","3.3.0-1"]},"glew":{"dependency_names":["glew"],"versions":["2.2.0-2","2.2.0-1","2.1.0-4","2.1.0-3","2.1.0-2","2.1.0-1"]},"glfw":{"dependency_names":["glfw3"],"versions":["3.4-2","3.4-1","3.3.10-1","3.3.9-1","3.3.8-3","3.3.8-2","3.3.8-1","3.3.7-1"]},"glib":{"dependency_names":["gthread-2.0","gobject-2.0","gmodule-no-export-2.0","gmodule-export-2.0","gmodule-2.0","glib-2.0","gio-2.0","gio-windows-2.0","gio-unix-2.0"],"program_names":["glib-genmarshal","glib-mkenums","glib-compile-schemas","glib-compile-resources","gio-querymodules","gdbus-codegen"],"versions":["2.88.2-1","2.88.1-1","2.88.0-1","2.86.4-1","2.86.3-1","2.86.2-1","2.86.1-1","2.86.0-1","2.84.4-1","2.84.3-1","2.84.2-1","2.84.1-1","2.84.0-1","2.82.5-1","2.82.4-1","2.82.2-1","2.82.1-1","2.82.0-1","2.80.5-1","2.80.4-1","2.80.3-1","2.80.2-1","2.80.0-1","2.78.4-1","2.78.3-1","2.78.2-1","2.78.1-1","2.78.0-1","2.76.5-1","2.76.4-1","2.76.3-1","2.76.1-1","2.74.4-1","2.74.1-1","2.74.0-2","2.74.0-1","2.72.2-1","2.72.1-1","2.70.4-1","2.70.2-1","2.70.1-1","2.70.0-1","2.68.1-1","2.66.7-1"]},"glib-networking":{"versions":["2.80.1-1","2.80.0-1","2.78.0-1"]},"glm":{"dependency_names":["glm"],"versions":["1.0.1-1","1.0.0-1","0.9.9.8-2","0.9.9.8-1"]},"glpk":{"dependency_names":["glpk"],"versions":["5.0-2","5.0-1"]},"gobject-introspection":{"dependency_names":["gobject-introspection-1.0"],"program_names":["g-ir-annotation-tool","g-ir-compiler","g-ir-generate","g-ir-inspect","g-ir-scanner","g-ir-doc-tool"],"versions":["1.86.0-1"]},"godot-cpp":{"dependency_names":["godot-cpp"],"versions":["4.5-1","4.4.1-2","4.4.1-1","4.3-1","4.2-1"]},"google-benchmark":{"dependency_names":["benchmark","benchmark_main"],"versions":["1.8.4-5","1.8.4-4","1.8.4-3","1.8.4-2","1.8.4-1","1.8.3-1","1.7.1-2","1.7.1-1","1.7.0-1","1.6.0-1","1.5.2-1","1.4.1-1"]},"google-brotli":{"dependency_names":["libbrotlicommon","libbrotlienc","libbrotlidec"],"program_names":["brotli"],"versions":["1.1.0-3","1.1.0-2","1.1.0-1","1.0.9-2","1.0.9-1","1.0.7-1"]},"google-snappy":{"dependency_names":["snappy"],"versions":["1.2.2-2","1.2.2-1","1.1.9-1","1.1.7-2","1.1.7-1"]},"google-woff2":{"dependency_names":["libwoff2common","libwoff2dec","libwoff2enc"],"program_names":["woff2_decompress","woff2_compress","woff2_info"],"versions":["1.0.2-4","1.0.2-3","1.0.2-2","1.0.2-1"]},"graphite2":{"dependency_names":["graphite2"],"versions":["1.3.14-1"]},"grpc":{"dependency_names":["grpc","grpc_unsecure","grpc++","grpc++_unsecure"],"program_names":["grpc_cpp_plugin","grpc_node_plugin","grpc_php_plugin","grpc_python_plugin","grpc_ruby_plugin"],"versions":["1.59.1-1"]},"gtest":{"dependency_names":["gtest","gtest_main","gmock","gmock_main"],"versions":["1.17.0-4","1.17.0-3","1.17.0-2","1.17.0-1","1.15.2-4","1.15.2-3","1.15.2-2","1.15.2-1","1.15.0-1","1.14.0-2","1.14.0-1","1.13.0-1","1.12.1-1","1.11.0-2","1.11.0-1","1.10.0-1","1.8.1-1","1.8.0-5","1.8.0-4","1.8.0-3","1.8.0-2","1.8.0-1","1.7.0-5","1.7.0-4","1.7.0-2"]},"gumbo-parser":{"dependency_names":["gumbo"],"versions":["0.12.1-2","0.12.1-1","0.10.1-1"]},"harfbuzz":{"dependency_names":["harfbuzz","harfbuzz-cairo","harfbuzz-gobject","harfbuzz-icu","harfbuzz-subset"],"versions":["13.0.1-1","12.3.2-1","12.3.1-1","12.3.0-1","12.2.0-1","12.1.0-1","11.5.0-1","11.4.5-1","11.4.4-1","11.4.3-1","11.4.1-1","11.3.3-1","11.3.2-1","11.2.1-1","11.2.0-1","11.1.0-1","11.0.1-1","11.0.0-1","10.4.0-1","10.3.0-1","10.2.0-1","10.1.0-1","10.0.1-1","9.0.0-1","8.3.1-1","8.3.0-1","8.2.2-1","8.2.1-1","8.2.0-1","8.1.1-1","5.2.0-1","4.4.1-1"]},"hedley":{"dependency_names":["hedley"],"versions":["15-1","11-1"]},"hinnant-date":{"dependency_names":["date","tz"],"versions":["3.0.3-1","3.0.1-2","3.0.1-1","3.0.0-1","2.4.1-1"]},"htslib":{"dependency_names":["htslib"],"versions":["1.22.1-1","1.22-1","1.21-2","1.21-1","1.20-1","1.17-2","1.17-1","1.16-3","1.16-2","1.16-1","1.15-1","1.14-1","1.11-1","1.10.2-1","1.9-1"]},"icu":{"dependency_names":["icu-data","icu-i18n","icu-io","icu-uc"],"program_names":["genbrk","genccode","gencmn"],"versions":["78.2-1","78.1-1","77.1-3","77.1-2","77.1-1","76.1-2","76.1-1","73.2-2","73.2-1","73.1-1","72.1-5","72.1-4","72.1-3","72.1-2","72.1-1","71.1-1","70.1-2","70.1-1","67.1-4","67.1-3","67.1-2","67.1-1","55.2-1"]},"iir":{"dependency_names":["iir"],"versions":["1.9.3-2","1.9.3-1","1.9.2-2","1.9.2-1"]},"imgui":{"dependency_names":["imgui"],"versions":["1.92.5-1","1.91.6-3","1.91.6-2","1.91.6-1","1.91.3-1","1.91.0-1","1.89.9-2","1.89.9-1","1.89.3-1","1.89.2-1","1.88-2","1.88-1","1.87-5","1.87-4","1.87-3","1.87-2","1.87-1","1.86-1","1.85-1","1.81-1","1.80-1","1.79-2","1.79-1","1.78-2","1.78-1","1.76-2","1.76-1"]},"imgui-docking":{"dependency_names":["imgui-docking"],"versions":["1.92.5-1","1.92.3-3","1.92.3-2","1.92.3-1","1.91.6-1","1.91.0-1"]},"imgui-sfml":{"dependency_names":["imgui-sfml"],"versions":["3.0-1","2.6.1-1","2.6-2","2.6-1","2.5-4","2.5-3","2.5-2","2.5-1","2.3-2","2.3-1","2.1-1"]},"imguizmo":{"dependency_names":["imguizmo"],"versions":["1.83-1"]},"implot":{"dependency_names":["implot"],"versions":["0.17-1","0.16-1"]},"indicators":{"dependency_names":["indicators"],"versions":["2.3-1","2.2-2","2.2-1"]},"inih":{"dependency_names":["inih","inireader"],"versions":["r62-1","r61-1","r59-1","r58-1","r57-1","r56-1","r54-1","r53-1","r52-1","r51-1"]},"irepeat":{"dependency_names":["irepeat"],"versions":["0.6-1"]},"jansson":{"dependency_names":["jansson"],"versions":["2.14.1-1","2.14-3","2.14-2","2.14-1","2.13-1","2.11-3","2.11-2","2.11-1"]},"jbig2dec":{"dependency_names":["jbig2dec"],"versions":["0.20-1"]},"jbigkit":{"dependency_names":["libjbig","libjbig85"],"program_names":["jbgtopbm","pbmtojbg","jbgtopbm85","pbmtojbg85"],"versions":["2.1-2","2.1-1"]},"jitterentropy":{"dependency_names":["jitterentropy"],"versions":["3.6.3-2","3.6.3-1"]},"json":{"versions":["3.2.0-1","2.1.1-1","2.0.5-1","2.0.3-1"]},"json-c":{"dependency_names":["json-c"],"versions":["0.18-2","0.18-1","0.17-2","0.17-1","0.16-4","0.16-3","0.16-2","0.16-1","0.15-2","0.15-1","0.13.1-1"]},"json-glib":{"dependency_names":["json-glib-1.0"],"versions":["1.10.8-1","1.10.6-1","1.10.0-1","1.6.6-2","1.6.6-1"]},"jsoncpp":{"dependency_names":["jsoncpp"],"versions":["1.9.8-1","1.9.7-1","1.9.6-1","1.9.5-2","1.9.5-1","1.8.4-1"]},"kafel":{"dependency_names":["kafel"],"program_names":["dump_policy_bpf"],"versions":["20231004-1"]},"kraken-engine":{"dependency_names":["kraken-engine"],"versions":["0.0.11-1","0.0.9-1","0.0.8-1","0.0.7-1","0.0.6-1","0.0.5-1","0.0.4-1","0.0.3-1","0.0.2-1","0.0.1-1"]},"lame":{"dependency_names":["mp3lame","mpglib"],"versions":["3.100-11","3.100-10","3.100-9","3.100-8","3.100-7","3.100-6","3.100-5","3.100-4","3.100-3","3.100-2","3.100-1","3.99.5-1"]},"lcms2":{"dependency_names":["lcms2"],"versions":["2.19.1-1","2.19-1","2.18-1","2.17-1","2.16-1","2.15-1","2.13.1-1","2.12-2","2.12-1"]},"leptonica":{"dependency_names":["lept"],"versions":["1.87.0-1","1.84.1-2","1.84.1-1","1.84.0-1","1.83.1-2","1.83.1-1"]},"libaegis":{"dependency_names":["libaegis"],"versions":["0.4.0-1"]},"libarchive":{"dependency_names":["libarchive"],"program_names":["bsdcat","bsdcpio","bsdtar"],"versions":["3.8.8-1","3.8.7-2","3.8.7-1","3.8.6-1","3.8.5-1","3.8.4-2","3.8.4-1","3.8.3-1","3.8.1-2","3.8.1-1","3.7.9-1","3.7.8-1","3.7.7-2","3.7.7-1","3.7.6-1","3.7.5-1","3.7.4-1","3.7.3-1","3.7.2-3","3.7.2-2","3.7.2-1","3.7.1-2","3.7.1-1","3.7.0-1","3.6.2-3","3.6.2-2","3.6.2-1"]},"libcap":{"dependency_names":["libcap"],"versions":["2.77-1","2.76-3","2.76-2","2.76-1"]},"libcbor":{"dependency_names":["libcbor"],"versions":["0.13.0-1"]},"libccp4c":{"dependency_names":["libccp4c"],"versions":["8.0.0-3","8.0.0-2","8.0.0-1","6.5.1-2","6.5.1-1"]},"libdicom":{"dependency_names":["libdicom"],"versions":["1.3.0-1","1.2.1-1","1.2.0-1","1.1.0-1","1.0.5-1","1.0.2-1","1.0.1-1"]},"libdrm":{"dependency_names":["libdrm","libdrm_amdgpu","libdrm_etnaviv","libdrm_exynos","libdrm_freedreno","libdrm_intel","libdrm_nouveau","libdrm_omap","libdrm_radeon","libdrm_tegra"],"versions":["2.4.134-1","2.4.133-1","2.4.131-1","2.4.130-1","2.4.129-1","2.4.128-1","2.4.127-1","2.4.126-1","2.4.125-1","2.4.124-1","2.4.123-1","2.4.121-1","2.4.115-1","2.4.114-1","2.4.111-1","2.4.110-1"]},"libebml":{"dependency_names":["libebml"],"versions":["1.4.5-3","1.4.5-2","1.4.5-1","1.4.4-2","1.4.4-1","1.4.3-1","1.4.2-1"]},"libexif":{"dependency_names":["libexif"],"versions":["0.6.25-1","0.6.24-5","0.6.24-4","0.6.24-3","0.6.24-2","0.6.24-1","0.6.22-1"]},"libffi":{"dependency_names":["libffi"],"versions":["3.6.0-1","3.5.2-2","3.5.2-1","3.5.1-2","3.5.1-1","3.4.8-1","3.4.6-3","3.4.6-2","3.4.6-1","3.4.4-4","3.4.4-3","3.4.4-2","3.4.4-1"]},"libffmpegthumbnailer":{"dependency_names":["libffmpegthumbnailer"],"versions":["2.2.4-1","2.2.3-2","2.2.3-1","2.2.2-4","2.2.2-3","2.2.2-2","2.2.2-1"]},"libgpiod":{"dependency_names":["libgpiod","libgpiodcxx"],"versions":["1.6.5-4","1.6.5-3","1.6.5-2","1.6.5-1","1.6.3-2","1.6.3-1"]},"libgrapheme":{"dependency_names":["libgrapheme"],"versions":["2.0.2-2","2.0.2-1"]},"libjpeg-turbo":{"dependency_names":["libjpeg","libturbojpeg"],"versions":["3.1.4.1-2","3.1.4.1-1","3.1.3-1","3.1.2-1","3.1.1-1","3.1.0-2","3.1.0-1","3.0.4-2","3.0.4-1","3.0.3-2","3.0.3-1","3.0.2-1","3.0.1-1","3.0.0-5","3.0.0-4","3.0.0-3","3.0.0-2","3.0.0-1","2.1.5.1-1","2.1.4-1","2.1.3-1","2.1.2-2","2.1.2-1","2.1.0-2","2.1.0-1"]},"libkqueue":{"dependency_names":["libkqueue"],"versions":["2.6.3-1","2.6.2-3","2.6.2-2","2.6.2-1","2.6.1-2","2.6.1-1"]},"liblangtag":{"dependency_names":["liblangtag"],"versions":["0.6.7-1","0.6.4-2","0.6.4-1","0.6.3-1"]},"liblastfm":{"dependency_names":["liblastfm"],"versions":["1.0.1-1"]},"liblbfgs":{"dependency_names":["liblbfgs"],"versions":["1.10-3","1.10-2","1.10-1"]},"libliftoff":{"dependency_names":["libliftoff"],"versions":["0.5.0-1","0.3.0-1"]},"liblzma":{"dependency_names":["liblzma"],"versions":["5.2.12-3","5.2.12-2","5.2.12-1","5.2.11-2","5.2.11-1","5.2.10-1","5.2.8-2","5.2.8-1","5.2.7-1","5.2.6-3","5.2.6-2","5.2.6-1","5.2.5-2","5.2.5-1","5.2.1-6","5.2.1-5","5.2.1-4","5.2.1-2"]},"libmatroska":{"dependency_names":["libmatroska"],"versions":["1.7.1-5","1.7.1-4","1.7.1-3","1.7.1-2","1.7.1-1","1.7.0-1","1.6.3-1"]},"libmicrohttpd":{"dependency_names":["libmicrohttpd"],"versions":["0.9.77-5","0.9.77-4","0.9.77-3","0.9.77-2","0.9.77-1","0.9.76-3","0.9.76-2","0.9.76-1"]},"libmysofa":{"dependency_names":["libmysofa"],"versions":["1.3.2-1"]},"libnpupnp":{"dependency_names":["libnpupnp"],"versions":["6.3.0-1","6.2.3-1","6.2.1-1","6.2.0-1","6.1.3-1","6.1.2-1","6.0.5-1","5.1.1-1","5.0.2-1","5.0.1-1","5.0.0-2","5.0.0-1","4.2.2-2","4.2.2-1","4.2.1-2","4.2.1-1"]},"libobsd":{"dependency_names":["libbsd-overlay","libobsd"],"versions":["1.1.1-1","1.1.0-1","1.0.0-1"]},"libopenjp2":{"dependency_names":["libopenjp2","libopenjpip"],"versions":["2.5.4-1","2.5.3-2","2.5.3-1","2.5.2-3","2.5.2-2","2.5.2-1","2.5.0-3","2.5.0-2","2.5.0-1","2.3.1-9","2.3.1-8","2.3.1-7","2.3.1-6","2.3.1-5","2.3.1-4","2.3.1-3","2.3.1-1"]},"libpfm":{"dependency_names":["libpfm"],"versions":["4.13.0-2","4.13.0-1"]},"libpng":{"dependency_names":["libpng"],"versions":["1.6.58-1","1.6.57-1","1.6.56-1","1.6.55-1","1.6.54-1","1.6.53-2","1.6.53-1","1.6.52-1","1.6.51-1","1.6.50-2","1.6.50-1","1.6.49-1","1.6.48-1","1.6.47-1","1.6.46-1","1.6.45-1","1.6.44-1","1.6.43-2","1.6.43-1","1.6.42-1","1.6.41-1","1.6.40-1","1.6.39-3","1.6.39-2","1.6.39-1","1.6.37-5","1.6.37-4","1.6.37-3","1.6.37-2","1.6.37-1","1.6.35-6","1.6.35-5","1.6.35-4","1.6.34-3","1.6.34-2","1.6.34-1","1.6.17-8","1.6.17-7","1.6.17-6","1.6.17-5","1.6.17-4","1.6.17-2"]},"libpsl":{"dependency_names":["libpsl"],"versions":["0.22.0-1","0.21.5-2","0.21.5-1","0.21.2-1"]},"libsass":{"dependency_names":["libsass"],"versions":["3.6.4-2","3.6.4-1"]},"libserialport":{"dependency_names":["libserialport"],"versions":["0.1.2-1"]},"libsigcplusplus-3":{"dependency_names":["sigc++-3.0"],"versions":["3.6.0-1"]},"libsignal-protocol-c":{"dependency_names":["libsignal-protocol-c"],"versions":["2.3.3-1"]},"libsndfile":{"dependency_names":["sndfile"],"versions":["1.2.2-2","1.2.2-1","1.2.0-1","1.1.0-6","1.1.0-5","1.1.0-4","1.1.0-3","1.1.0-2","1.1.0-1"]},"libsrtp2":{"dependency_names":["libsrtp2"],"versions":["2.8.0-1","2.7.0-1","2.6.0-1","2.5.0-1","2.4.2-1","2.4.0-1","2.2.0-1"]},"libssh2":{"dependency_names":["libssh2"],"versions":["1.11.1-2","1.11.1-1","1.11.0-1","1.10.0-5","1.10.0-4","1.10.0-3","1.10.0-2","1.10.0-1"]},"libtiff":{"dependency_names":["libtiff-4"],"versions":["4.7.1-5","4.7.1-4","4.7.1-3","4.7.1-2","4.7.1-1","4.7.0-1","4.6.0-1","4.5.1-1","4.5.0-3","4.5.0-2","4.5.0-1","4.4.0-3","4.4.0-2","4.4.0-1","4.1.0-5","4.1.0-4","4.1.0-3","4.1.0-2","4.1.0-1"]},"libtirpc":{"dependency_names":["libtirpc"],"versions":["1.3.7-1","1.3.3-1"]},"libtomcrypt":{"dependency_names":["libtomcrypt"],"versions":["1.18.2-1","1.17-1"]},"libunibreak":{"dependency_names":["libunibreak"],"versions":["6.1-1","5.1-3","5.1-2","5.1-1"]},"libupnp":{"dependency_names":["libupnp"],"versions":["1.14.27-1","1.14.25-3","1.14.25-2","1.14.25-1","1.14.24-1","1.14.23-1","1.14.20-2","1.14.20-1","1.14.19-1","1.14.18-1","1.14.17-1","1.14.15-1","1.14.14-2","1.14.14-1","1.14.13-1","1.14.12-2","1.14.12-1"]},"liburing":{"dependency_names":["liburing"],"versions":["2.14-1","2.12-1","2.5-2","2.5-1","2.4-2","2.4-1","2.3-3","2.3-2","2.3-1","2.2-2","2.2-1","2.1-1","2.0-1"]},"libusb":{"dependency_names":["libusb-1.0"],"versions":["1.0.29-2","1.0.29-1","1.0.28-1","1.0.27-3","1.0.27-2","1.0.27-1","1.0.26-5","1.0.26-4","1.0.26-3","1.0.26-2","1.0.26-1"]},"libuv":{"dependency_names":["libuv"],"versions":["1.51.0-1","1.49.1-1","1.48.0-1","1.47.0-1","1.46.0-1","1.44.2-2","1.44.2-1","1.44.1-1","1.43.0-1","1.42.0-1","1.41.0-1","1.18.0-3","1.18.0-2","1.18.0-1"]},"libwebp":{"dependency_names":["libsharyuv","libwebp","libwebpdecoder","libwebpdemux","libwebpmux"],"program_names":["cwebp","dwebp","webpinfo","webpmux"],"versions":["1.6.0-1","1.5.0-1","1.3.2-2","1.3.2-1","1.3.1-2","1.3.1-1"]},"libwebsockets":{"dependency_names":["libwebsockets"],"versions":["4.3.6-1","4.3.5-1","4.3.3-1","4.3.2-3","4.3.2-2","4.3.2-1","4.0.21-2","4.0.21-1","4.0.13-1"]},"libxcursor":{"dependency_names":["xcursor"],"versions":["1.2.3-1","1.2.1-1"]},"libxext":{"dependency_names":["xext"],"versions":["1.3.7-1","1.3.6-1","1.3.5-1","1.3.4-1"]},"libxinerama":{"dependency_names":["xinerama"],"versions":["1.1.6-1","1.1.5-1","1.1.4-1"]},"libxkbcommon":{"dependency_names":["xkbcommon","xkbcommon-x11","xkbregistry"],"versions":["1.11.0-2","1.11.0-1","1.10.0-1","1.9.2-1","1.9.1-1","1.8.1-1","1.7.0-1"]},"libxml2":{"dependency_names":["libxml-2.0"],"versions":["2.15.3-1","2.15.2-1","2.15.1-1","2.15.0-1","2.14.6-1","2.14.5-1","2.14.4-1","2.14.3-1","2.14.2-1","2.14.1-1","2.13.6-1","2.13.5-1","2.13.4-1","2.13.3-1","2.13.2-1","2.13.1-1","2.13.0-1","2.12.7-1","2.12.6-1","2.12.5-1","2.11.6-3","2.11.6-2","2.11.6-1","2.11.5-2","2.11.5-1","2.11.4-1","2.11.1-1","2.10.4-1","2.10.3-4","2.10.3-3","2.10.3-2","2.10.3-1","2.10.2-2","2.10.2-1","2.9.14-1","2.9.7-14","2.9.7-13","2.9.7-12","2.9.7-11","2.9.7-10","2.9.7-9","2.9.7-8","2.9.7-7","2.9.7-6","2.9.7-5","2.9.7-4","2.9.7-3","2.9.7-2","2.9.7-1","2.9.2-5","2.9.2-4","2.9.2-2"]},"libxmlpp":{"dependency_names":["libxml++-5.0"],"versions":["5.6.1-1","5.6.0-1","5.4.0-2","5.4.0-1","5.2.0-1","5.0.3-1","5.0.2-2","5.0.2-1","5.0.1-1","3.0.1-2","3.0.1-1"]},"libxrandr":{"dependency_names":["xrandr"],"versions":["1.5.5-1","1.5.4-1","1.5.2-1"]},"libxrender":{"dependency_names":["xrender"],"versions":["0.9.12-1","0.9.10-1"]},"libxslt":{"dependency_names":["libxslt","libexslt"],"program_names":["xsltproc"],"versions":["1.1.45-1","1.1.43-3","1.1.43-2","1.1.43-1","1.1.39-1","1.1.38-1","1.1.37-4","1.1.37-3","1.1.37-2","1.1.37-1","1.1.36-1","1.1.35-1","1.1.34-1"]},"libxv":{"dependency_names":["xv"],"versions":["1.0.13-1","1.0.11-1"]},"libxxf86vm":{"dependency_names":["xxf86vm"],"versions":["1.1.7-1","1.1.6-1","1.1.4-1"]},"libyaml":{"dependency_names":["yaml-0.1"],"versions":["0.2.5-1"]},"littlefs":{"dependency_names":["littlefs"],"versions":["2.11.2-1","2.11.0-1","2.10.2-1","2.10.1-1","2.9.3-1","2.9.0-1"]},"lmdb":{"dependency_names":["lmdb"],"program_names":["mdb_stat","mdb_copy","mdb_dump","mdb_load"],"versions":["0.9.33-1","0.9.29-3","0.9.29-2","0.9.29-1"]},"lua":{"dependency_names":["lua-5.5","lua"],"versions":["5.5.0-1","5.4.8-1","5.4.6-5","5.4.6-4","5.4.6-3","5.4.6-2","5.4.6-1","5.4.4-1","5.4.3-2","5.4.3-1","5.3.6-2","5.3.6-1","5.3.0-6","5.3.0-5","5.3.0-4","5.3.0-2"]},"luajit":{"dependency_names":["luajit"],"program_names":["luajit"],"versions":["2.1.1720049189-3","2.1.1720049189-2","2.1.1720049189-1","2.1.1713773202-1"]},"ludocode-mpack":{"dependency_names":["ludocode-mpack"],"versions":["1.1.1-1","1.1-1","1.0-1"]},"lvgl":{"dependency_names":["lvgl","lvgl_demos","lvgl_examples","lvgl_thorvg"],"versions":["9.5.0-1","9.4.0-1","9.3.0-1","9.2.2-2","9.2.2-1"]},"lz4":{"dependency_names":["liblz4"],"versions":["1.10.0-2","1.10.0-1","1.9.4-2","1.9.4-1","1.9.3-1","1.9.2-1"]},"lzo2":{"dependency_names":["lzo2"],"versions":["2.10-4","2.10-3","2.10-2","2.10-1"]},"m4":{"program_names":["m4"],"versions":["1.4.19-2","1.4.19-1"]},"magic_enum":{"dependency_names":["magic_enum"],"versions":["0.9.8-1","0.9.7-1","0.9.6-1","0.9.5-1","0.9.3-1","0.9.2-1","0.8.2-1","0.8.1-1"]},"mdds":{"dependency_names":["mdds-2.1"],"versions":["2.1.1-1","2.0.1-1","1.6.0-2","1.6.0-1"]},"microsoft-gsl":{"dependency_names":["msft_gsl"],"versions":["4.2.1-1","4.2.0-1","4.0.0-1","3.1.0-1","2.0.0-1"]},"mimalloc":{"dependency_names":["mimalloc"],"versions":["3.3.2-1","3.2.7-1","3.1.5-1"]},"miniaudio":{"dependency_names":["miniaudio"],"versions":["0.11.22-2","0.11.22-1","0.11.21-2","0.11.21-1"]},"minifycpp":{"dependency_names":["minifycpp"],"versions":["0.2.4-1","0.1.2-1"]},"miniz":{"dependency_names":["miniz"],"versions":["3.1.0-1","3.0.2-1","3.0.1-1","2.2.0-1","2.1.0-2","2.1.0-1"]},"minizip-ng":{"dependency_names":["minizip"],"versions":["4.0.10-3","4.0.10-2","4.0.10-1","4.0.9-2","4.0.9-1","4.0.7-1","4.0.4-1","4.0.1-2","4.0.1-1","4.0.0-2","4.0.0-1","3.0.10-1","3.0.8-1","3.0.7-1","3.0.6-1"]},"mldsa-native":{"dependency_names":["mldsa-native"],"versions":["1.0.0-1"]},"mocklibc":{"versions":["1.0-2"]},"mpark-patterns":{"dependency_names":["mpark_patterns"],"versions":["0.3.0-1"]},"mpdecimal":{"dependency_names":["mpdec","mpdecpp"],"versions":["2.5.1-4","2.5.1-3","2.5.1-2","2.5.1-1","2.5.0-2","2.5.0-1"]},"msgpackc-cxx":{"dependency_names":["msgpack-cxx"],"versions":["7.0.0-1","6.1.1-2","6.1.1-1","6.1.0-2","6.1.0-1","5.0.0-1"]},"mt32emu":{"dependency_names":["mt32emu"],"versions":["2.7.2-1","2.7.1-1","2.7.0-1","2.6.1-1","2.5.3-1","2.5.0-1","2.4.2-2","2.4.2-1"]},"muellan-clipp":{"versions":["1.2.3-1"]},"mujs":{"dependency_names":["mujs"],"program_names":["mujs","mujs-pp"],"versions":["1.3.8-1","1.3.6-1","1.3.5-1","1.3.3-1"]},"nanoarrow":{"dependency_names":["nanoarrow","nanoarrow-ipc"],"versions":["0.8.0-1","0.7.0-1","0.6.0-1","0.5.0-1"]},"nanobind":{"dependency_names":["nanobind"],"versions":["2.12.0-1","2.10.2-1","2.9.2-1","2.8.0-1","2.7.0-4","2.7.0-3","2.7.0-2","2.7.0-1","2.4.0-2","2.4.0-1","2.2.0-2","2.2.0-1","2.1.0-1"]},"nativefiledialog-extended":{"dependency_names":["nativefiledialog-extended"],"versions":["1.2.1-1","1.1.1-2","1.1.1-1"]},"netstring-c":{"dependency_names":["netstring-c"],"versions":["0.0.0-4","0.0.0-3","0.0.0-2","0.0.0-1"]},"nghttp2":{"dependency_names":["libnghttp2"],"versions":["1.62.1-3","1.62.1-2","1.62.1-1","1.58.0-1","1.56.0-1"]},"nlohmann_json":{"dependency_names":["nlohmann_json"],"versions":["3.12.0-1","3.11.3-1","3.11.2-1","3.10.5-1","3.9.1-1","3.7.0-1","3.4.0-2","3.4.0-1","3.3.0-1"]},"nng":{"dependency_names":["nng"],"versions":["1.11-1","1.5.2-4","1.5.2-3","1.5.2-2","1.5.2-1","1.3.2-2","1.3.2-1"]},"nonstd-any-lite":{"versions":["0.2.0-1"]},"nonstd-byte-lite":{"versions":["0.2.0-1"]},"nonstd-expected-lite":{"dependency_names":["expected-lite"],"versions":["0.3.0-2","0.3.0-1"]},"nonstd-observer-ptr-lite":{"versions":["0.4.0-1"]},"nonstd-optional-lite":{"versions":["3.2.0-1"]},"nonstd-span-lite":{"versions":["0.5.0-1"]},"nonstd-status-value-lite":{"versions":["1.1.0-1"]},"nonstd-string-view-lite":{"versions":["1.3.0-1"]},"nowide":{"dependency_names":["nowide"],"versions":["11.3.0-1","11.2.0-1"]},"oatpp":{"dependency_names":["oatpp","oatpp-test"],"versions":["1.3.1-2","1.3.1-1","1.3.0-2","1.3.0-1"]},"oatpp-openssl":{"dependency_names":["oatpp-openssl"],"versions":["1.3.0-2","1.3.0-1"]},"oatpp-sqlite":{"dependency_names":["oatpp-sqlite"],"versions":["1.3.0-1"]},"oatpp-swagger":{"dependency_names":["oatpp-swagger"],"versions":["1.3.0-1"]},"oatpp-websocket":{"dependency_names":["oatpp-websocket"],"versions":["1.3.0-1"]},"oatpp-zlib":{"dependency_names":["oatpp-zlib"],"versions":["1.3.0-1"]},"ogg":{"dependency_names":["ogg"],"versions":["1.3.6-1","1.3.5-6","1.3.5-5","1.3.5-4","1.3.5-3","1.3.5-2","1.3.5-1","1.3.2-7","1.3.2-6","1.3.2-5","1.3.2-4","1.3.2-2"]},"onqtam-doctest":{"versions":["2.4.0-1","2.3.7-1"]},"openal-soft":{"dependency_names":["openal"],"versions":["1.24.3-2","1.24.3-1","1.24.2-1","1.24.1-1","1.24.0-1","1.23.1-3","1.23.1-2","1.23.1-1","1.23.0-1","1.22.2-8","1.22.2-7","1.22.2-6","1.22.2-5","1.22.2-4","1.22.2-3","1.22.2-2","1.22.2-1","1.21.0-1"]},"openblas":{"dependency_names":["openblas"],"versions":["0.3.28-2","0.3.28-1"]},"opencl-headers":{"dependency_names":["opencl-headers"],"versions":["2024.10.24-1","2023.04.17-1","2022.05.18-1","2021.06.30-1"]},"openh264":{"dependency_names":["openh264"],"versions":["2.6.0-1","2.5.0-1","2.4.1-1","2.3.1-1","1.7.0-1"]},"openssl":{"dependency_names":["libcrypto","libssl","openssl"],"versions":["3.0.8-3","3.0.8-2","3.0.8-1","3.0.7-2","3.0.7-1","3.0.2-1","1.1.1l-3","1.1.1l-2","1.1.1l-1","1.1.1k-2","1.1.1k-1"]},"opus":{"dependency_names":["opus"],"versions":["1.6.1-1","1.6-1","1.5.2-1","1.4-1"]},"orcus":{"dependency_names":["liborcus-0.19"],"versions":["0.19.2-1","0.17.2-3","0.17.2-2","0.17.2-1"]},"orocos_kdl":{"dependency_names":["orocos-kdl"],"versions":["1.5.1-1"]},"pango":{"dependency_names":["pango","pangoft2","pangoxft","pangowin32","pangocairo"],"versions":["1.56.4-1","1.56.3-1","1.56.2-1","1.56.1-1","1.56.0-1","1.51.0-2","1.51.0-1"]},"pcg":{"versions":["0.98.1-2","0.98.1-1"]},"pciaccess":{"dependency_names":["pciaccess"],"versions":["0.19-1","0.18.1-2","0.18.1-1","0.18-1"]},"pcre":{"dependency_names":["libpcre","libpcreposix"],"versions":["8.45-5","8.45-4","8.45-3","8.45-2","8.45-1","8.37-4","8.37-3","8.37-2","8.37-1"]},"pcre2":{"dependency_names":["libpcre2-8","libpcre2-16","libpcre2-32","libpcre2-posix"],"versions":["10.47-3","10.47-2","10.47-1","10.46-1","10.45-4","10.45-3","10.45-2","10.45-1","10.44-2","10.44-1","10.43-1","10.42-5","10.42-4","10.42-3","10.42-2","10.42-1","10.40-3","10.40-2","10.40-1","10.39-3","10.39-2","10.39-1","10.23-1"]},"pegtl":{"dependency_names":["pegtl"],"versions":["3.2.8-1","3.2.7-1"]},"physfs":{"dependency_names":["physfs"],"versions":["3.2.0-2","3.2.0-1"]},"pixman":{"dependency_names":["pixman-1"],"versions":["0.46.4-1","0.46.2-1","0.46.0-1","0.44.2-1","0.44.0-1","0.43.4-1","0.43.2-1","0.42.2-1"]},"pkgconf":{"dependency_names":["libpkgconf"],"versions":["2.5.1-1","2.5.0-1","2.4.3-1","2.3.0-1","2.1.1-1","1.9.3-2","1.9.3-1","1.9.2-1","1.9.0-1"]},"protobuf":{"dependency_names":["protobuf-lite","protobuf","protoc"],"program_names":["protoc"],"versions":["25.2-4","25.2-3","25.2-2","25.2-1","3.21.12-5","3.21.12-4","3.21.12-3","3.21.12-2","3.21.12-1","3.21.9-1","3.20.1-1","3.20.0-1","3.12.2-2","3.12.2-1","3.5.1-3","3.5.1-2","3.5.0-4","3.5.0-3","3.5.0-2"]},"protobuf-c":{"dependency_names":["libprotobuf-c"],"versions":["1.5.0-1","1.4.1-3","1.4.1-2","1.4.1-1","1.3.0-1"]},"proxy-libintl":{"dependency_names":["intl"],"versions":["0.5-1","0.4-1"]},"pugixml":{"dependency_names":["pugixml"],"versions":["1.15-1","1.14-1","1.13-3","1.13-2","1.13-1","1.12.1-1","1.12-1","1.11.4-1"]},"pv":{"program_names":["pv"],"versions":["1.10.5-1","1.10.3-1","1.10.2-1","1.9.34-2","1.9.34-1","1.9.31-1","1.8.14-1","1.8.13-1","1.8.12-1","1.8.10-1","1.8.9-1","1.8.5-1"]},"pybind11":{"dependency_names":["pybind11"],"versions":["3.0.0-1","2.13.5-1","2.13.1-1","2.13.0-1","2.12.0-1","2.11.1-1","2.10.4-1","2.10.3-1","2.10.0-1","2.9.0-1","2.6.1-1","2.6.0-1","2.3.0-2","2.3.0-1","2.2.4-1","2.2.3-1","2.2.2-1"]},"qarchive":{"dependency_names":["qarchive"],"versions":["2.2.7-1","2.2.6-1","2.2.4-1","2.2.3-1","2.2.2-1","2.2.1-2","2.2.1-1","2.2.0-1","2.0.2-1"]},"qrencode":{"dependency_names":["libqrencode"],"versions":["4.1.1-3","4.1.1-2","4.1.1-1"]},"quazip":{"dependency_names":["quazip"],"versions":["1.4-1","1.3-1","0.7.3-1"]},"quickjs-ng":{"dependency_names":["quickjs-ng"],"program_names":["qjsc","qjs"],"versions":["0.15.1-1","0.15.0-1","0.14.0-1","0.13.0-1","0.12.1-1","0.12.0-1","0.11.0-1","0.10.1-1","0.10.0-1","0.9.0-1"]},"quill":{"dependency_names":["quill"],"versions":["12.0.0-1","11.1.0-1","11.0.2-1","11.0.1-1","10.2.0-1","10.1.0-1","10.0.1-1","10.0.0-1","9.0.3-1","9.0.2-1","9.0.0-1","8.2.0-1","8.1.1-1","8.1.0-1","8.0.0-1","7.5.0-1","7.4.0-1","7.3.0-1","7.2.2-1","7.1.0-1","7.0.0-1","6.0.0-1","4.5.0-1","4.4.0-1","4.2.0-1"]},"rang":{"dependency_names":["rang"],"versions":["3.2-1","2.0-1"]},"range-v3":{"dependency_names":["range-v3"],"versions":["0.12.0-2","0.12.0-1","0.11.0-1","0.10.0-1","0.3.6-1"]},"rapidjson":{"dependency_names":["rapidjson"],"versions":["1.1.0-2","1.1.0-1"]},"rdkafka":{"dependency_names":["rdkafka","rdkafka++"],"versions":["2.10.0-1","2.5.3-1","2.3.0-1","2.1.0-2","2.1.0-1","1.9.2-5","1.9.2-4","1.9.2-3","1.9.2-2","1.9.2-1","1.9.0-2","1.9.0-1","1.8.2-1","1.7.0-1","1.5.2-1","1.5.0-1","1.4.4-1","1.4.0-1"]},"re2":{"dependency_names":["re2"],"versions":["20230301-3","20230301-2","20230301-1","20220401-1","20201101-1"]},"reflex":{"dependency_names":["reflex"],"program_names":["reflex"],"versions":["5.1.1-3","5.1.1-2","5.1.1-1","5.1.0-2","5.1.0-1","3.2.11-1","3.2.7-1","3.2.3-1","3.0.1-2","3.0.1-1"]},"robin-map":{"dependency_names":["robin-map"],"versions":["1.4.0-1","1.3.0-1","1.2.1-1"]},"rtaudio":{"dependency_names":["rtaudio"],"versions":["6.0.1-1","5.2.0-1"]},"rubberband":{"dependency_names":["rubberband"],"versions":["4.0.0-1","3.3.0-1","2.0.2-1"]},"rxcpp":{"dependency_names":["rxcpp"],"versions":["4.1.1-1","4.1.0-1"]},"s2n-tls":{"dependency_names":["s2n-tls"],"versions":["1.5.27-1"]},"sassc":{"program_names":["sassc"],"versions":["3.6.2-2","3.6.2-1"]},"scdoc":{"program_names":["scdoc"],"versions":["1.11.4-1"]},"sdl2":{"dependency_names":["sdl2","sdl2main","sdl2_test"],"versions":["2.32.8-1","2.30.6-2","2.30.6-1","2.30.3-3","2.30.3-2","2.30.3-1","2.28.5-2","2.28.5-1","2.28.1-2","2.28.1-1","2.26.5-5","2.26.5-4","2.26.5-3","2.26.5-2","2.26.5-1","2.26.0-2","2.26.0-1","2.24.2-1","2.24.1-4","2.24.1-3","2.24.1-2","2.24.1-1","2.24.0-5","2.24.0-4","2.24.0-3","2.24.0-2","2.24.0-1","2.0.20-6","2.0.20-5","2.0.20-4","2.0.20-3","2.0.20-2","2.0.20-1","2.0.18-2","2.0.18-1","2.0.12-3","2.0.12-2","2.0.12-1","2.0.3-6","2.0.3-5","2.0.3-4"]},"sdl2_image":{"dependency_names":["sdl2_image"],"versions":["2.6.3-3","2.6.3-2","2.6.3-1","2.6.2-1","2.0.5-3","2.0.5-2","2.0.5-1"]},"sdl2_mixer":{"dependency_names":["sdl2_mixer"],"versions":["2.6.2-4","2.6.2-3","2.6.2-2","2.6.2-1","2.0.4-3","2.0.4-2","2.0.4-1"]},"sdl2_net":{"dependency_names":["sdl2_net"],"versions":["2.2.0-3","2.2.0-2","2.2.0-1","2.0.1-4","2.0.1-3","2.0.1-2","2.0.1-1"]},"sdl2_ttf":{"dependency_names":["sdl2_ttf"],"versions":["2.20.1-4","2.20.1-3","2.20.1-2","2.20.1-1","2.0.12-3","2.0.12-2","2.0.12-1"]},"sdl3":{"dependency_names":["sdl3"],"versions":["3.4.2-1","3.4.0-2","3.4.0-1","3.2.10-2","3.2.10-1","3.2.4-3","3.2.4-2"]},"sdl3_image":{"dependency_names":["sdl3_image"],"versions":["3.2.4-1"]},"sdl3_ttf":{"dependency_names":["sdl3_ttf"],"versions":["3.2.2-2"]},"sds":{"dependency_names":["sds"],"versions":["2.0.0-2","2.0.0-1"]},"sergiusthebest-plog":{"dependency_names":["plog"],"versions":["1.1.10-2","1.1.10-1","1.1.4-1"]},"sfml":{"dependency_names":["sfml","sfml-all"],"versions":["3.0.2-1","3.0.1-1","2.6.2-2","2.6.2-1","2.6.1-1","2.6.0-2","2.6.0-1","2.5.1-4","2.5.1-3","2.5.1-2","2.5.1-1"]},"simdjson":{"dependency_names":["simdjson"],"versions":["4.2.2-1","3.13.0-1","3.12.3-1","3.12.2-1","3.11.3-1","3.10.1-1","3.3.0-2","3.3.0-1","3.1.1-1"]},"slirp":{"dependency_names":["slirp"],"versions":["4.9.3-1","4.9.1-1","4.9.0-1","4.8.0-1","4.7.0-1","4.6.1-2","4.6.1-1"]},"snitch":{"dependency_names":["snitch"],"versions":["1.3.2-1"]},"soundtouch":{"dependency_names":["soundtouch"],"versions":["2.4.1-1","2.4.0-1","2.3.2-5","2.3.2-4","2.3.2-3","2.3.2-2","2.3.2-1"]},"sparkcli":{"dependency_names":["sparkcli"],"versions":["0.0.6-1","0.0.4-1","0.0.1-1"]},"sparrow":{"dependency_names":["sparrow"],"versions":["0.0.2-1"]},"sparsehash-c11":{"dependency_names":["sparsehash-c11"],"versions":["2.11.1-1"]},"spdlog":{"dependency_names":["spdlog"],"versions":["1.17.0-1","1.15.3-5","1.15.3-4","1.15.3-3","1.15.3-2","1.15.3-1","1.15.2-3","1.15.2-2","1.15.2-1","1.15.1-1","1.15.0-1","1.14.1-2","1.14.1-1","1.14.0-1","1.13.0-4","1.13.0-3","1.13.0-2","1.13.0-1","1.12.0-2","1.12.0-1","1.11.0-2","1.11.0-1","1.10.0-3","1.10.0-2","1.10.0-1","1.9.2-1","1.8.5-1","1.8.2-1","1.8.1-1","1.8.0-1","1.4.2-1","1.3.1-1","1.1.0-1","0.17.0-2","0.17.0-1","0.16.3-1","0.11.0-1"]},"speexdsp":{"dependency_names":["speexdsp"],"versions":["1.2.1-8","1.2.1-7","1.2.1-6","1.2.1-5","1.2.1-4","1.2.1-3","1.2.1-2","1.2.1-1","1.2.0-1"]},"spng":{"dependency_names":["spng"],"versions":["0.7.4-1","0.7.2-1","0.7.0-1","0.5.0-1","0.4.5-1","0.4.4-1","0.4.3-1","0.4.2-2","0.4.2-1"]},"sqlite3":{"dependency_names":["sqlite3"],"versions":["3.53.3-1","3.53.2-1","3.53.1-1","3.53.0-1","3.52.0-1","3.51.2-1","3.51.1-1","3.51.0-1","3.50.4-1","3.50.3-1","3.50.2-1","3.50.1-1","3.50.0-1","3.49.2-1","3.49.1-1","3.49.0-1","3.48.0-1","3.47.2-2","3.47.2-1","3.47.1-1","3.47.0-1","3.46.1-3","3.46.1-2","3.46.1-1","3.46.0-1","3.45.3-1","3.45.2-1","3.45.1-1","3.45.0-1","3.44.2-1","3.44.1-1","3.44.0-1","3.43.2-1","3.43.1-2","3.43.1-1","3.43.0-1","3.42.0-1","3.41.2-2","3.41.2-1","3.41.1-1","3.41.0-1","3.40.0-1","3.39.4-2","3.39.4-1","3.39.3-1","3.38.0-1","3.34.1-1"]},"sqlitecpp":{"dependency_names":["sqlitecpp"],"versions":["3.3.2-1","3.3.1-1"]},"sqlpp11":{"dependency_names":["sqlpp11"],"program_names":["ddl2cpp"],"versions":["0.64-1"]},"stc":{"dependency_names":["stc"],"program_names":["checkscoped"],"versions":["5.0-2","5.0-1"]},"stduuid":{"dependency_names":["stduuid"],"versions":["1.2.3-2","1.2.3-1"]},"svtjpegxs":{"dependency_names":["SvtJpegxs"],"versions":["0.9.0-2","0.9.0-1"]},"tabulate":{"dependency_names":["tabulate"],"versions":["1.5-1","1.4-2","1.4-1"]},"taglib":{"dependency_names":["taglib"],"versions":["2.1.1-1","2.0.2-1","2.0-1","1.13.1-1","1.13-4","1.13-3","1.13-2","1.13-1","1.12-3","1.12-2","1.12-1"]},"tclap":{"dependency_names":["tclap"],"versions":["1.2.4-2","1.2.4-1","1.2.2-1","1.2.1-1"]},"termbox":{"dependency_names":["termbox"],"versions":["1.1.2-2","1.1.2-1"]},"termbox2":{"dependency_names":["termbox2"],"versions":["2.5.0-2","2.5.0-1"]},"theora":{"dependency_names":["theoraenc","theoradec","theora"],"versions":["1.2.0-1","1.1.1-6","1.1.1-5","1.1.1-4","1.1.1-3","1.1.1-2","1.1.1-1"]},"tinyfsm":{"dependency_names":["tinyfsm"],"versions":["0.3.3-2","0.3.3-1"]},"tinyply":{"dependency_names":["tinyply"],"versions":["2.3.4-1","2.2-2","2.2-1","2.1-2","2.1-1","2.0-1"]},"tinyxml2":{"dependency_names":["tinyxml2"],"versions":["11.0.0-2","11.0.0-1","10.1.0-1","10.0.0-1","9.0.0-1","7.0.1-1"]},"tkrzw":{"dependency_names":["tkrzw"],"program_names":["tkrzw_dbm_perf","tkrzw_dbm_util","tkrzw_ulog_util"],"versions":["1.0.32-1"]},"tl-expected":{"dependency_names":["tl-expected"],"versions":["1.3.1-1","1.1.0-1","1.0.0-1"]},"tl-optional":{"dependency_names":["tl-optional"],"versions":["1.1.0-1","1.0.0-1"]},"tomlplusplus":{"dependency_names":["tomlplusplus"],"versions":["3.4.0-1","3.3.0-1","3.2.0-1","3.1.0-1","3.0.0-1","2.5.0-1"]},"tracy":{"dependency_names":["tracy"],"versions":["0.13.0-1","0.12.2-1","0.10-1","0.9.1-1","0.9-1","0.8.2.1-1","0.8.1-1"]},"tree-sitter":{"dependency_names":["tree-sitter"],"versions":["0.26.3-1","0.22.5-3","0.22.5-2","0.22.5-1"]},"trompeloeil":{"dependency_names":["trompeloeil"],"versions":["49-1","48-1","47-1","39-1","38-1"]},"tronkko-dirent":{"versions":["1.23.2-1"]},"turtle":{"versions":["1.3.2-1","1.3.1-1"]},"type_safe":{"dependency_names":["type_safe"],"versions":["0.2.4-1","0.2.3-3","0.2.3-2","0.2.3-1"]},"uchardet":{"dependency_names":["uchardet"],"versions":["0.0.8-1","0.0.7-3","0.0.7-2","0.0.7-1"]},"unit-system":{"dependency_names":["unit-system"],"versions":["0.7.1-1","0.7.0-1","0.6.1-1","0.5.2-1","0.5.0-1"]},"unittest-cpp":{"dependency_names":["unittest-cpp"],"versions":["2.0.0-2","2.0.0-1"]},"unity":{"dependency_names":["unity"],"versions":["2.6.1-1","2.5.2-1"]},"usbbluetooth":{"dependency_names":["usbbluetooth"],"versions":["0.0.9-1"]},"utf8proc":{"dependency_names":["libutf8proc"],"versions":["2.10.0-1","2.9.0-1","2.8.0-1","2.7.0-1","2.6.0-1"]},"utfcpp":{"dependency_names":["utf8cpp"],"versions":["4.0.8-1","4.0.6-1","4.0.5-1","3.2.4-1","3.2.3-1","3.2.2-1","3.2.1-1","2.3.5-1"]},"uthash":{"dependency_names":["uthash"],"versions":["2.4.0-1","2.3.0-2","2.3.0-1"]},"vo-aacenc":{"dependency_names":["vo-aacenc"],"versions":["0.1.3-2","0.1.3-1"]},"vorbis":{"dependency_names":["vorbis","vorbisfile","vorbisenc"],"versions":["1.3.7-4","1.3.7-3","1.3.7-2","1.3.7-1","1.3.7-0","1.3.5-8","1.3.5-7","1.3.5-6","1.3.5-5","1.3.5-4","1.3.5-2"]},"vulkan-headers":{"dependency_names":["VulkanHeaders"],"versions":["1.4.346-1","1.3.283-1","1.3.265-1","1.2.203-1","1.2.158-2","1.2.158-1","1.2.142-1"]},"vulkan-memory-allocator":{"dependency_names":["VulkanMemoryAllocator"],"versions":["3.3.0-1","3.1.0-1","3.0.1-1"]},"vulkan-validationlayers":{"versions":["1.2.158-2","1.2.158-1"]},"wayland":{"dependency_names":["wayland-client","wayland-cursor","wayland-egl","wayland-server"],"program_names":["wayland-scanner"],"versions":["1.25.0-1","1.24.0-2","1.24.0-1","1.23.1-1","1.23.0-1","1.20.0-1"]},"wayland-protocols":{"dependency_names":["wayland-protocols"],"versions":["1.49-1","1.48-1","1.47-1","1.46-1","1.45-1","1.44-1","1.43-1","1.42-1","1.41-1","1.40-1","1.39-1","1.38-1","1.37-1","1.35-1","1.32-1","1.24-1"]},"websocketpp":{"dependency_names":["websocketpp"],"versions":["0.8.2-2","0.8.2-1"]},"win-iconv":{"dependency_names":["iconv"],"program_names":["win_iconv"],"versions":["0.0.9-1","0.0.8-1"]},"wren":{"dependency_names":["wren"],"versions":["0.4.0-3","0.4.0-2","0.4.0-1","0.3.0-1"]},"x-plane-sdk":{"dependency_names":["xplm","xpwidgets","xpcpp"],"versions":["4.2.0-1","4.0.1-1"]},"xdelta3":{"program_names":["xdelta3"],"versions":["3.2.0-1","3.1.0-1"]},"xtensor":{"dependency_names":["xtensor"],"versions":["0.21.5-1"]},"xtl":{"dependency_names":["xtl"],"versions":["0.8.0-1","0.6.12-1"]},"xvidcore":{"dependency_names":["xvidcore"],"versions":["1.3.7-1"]},"xxhash":{"dependency_names":["libxxhash"],"versions":["0.8.3-2","0.8.3-1","0.8.2-1","0.8.1-2","0.8.1-1","0.8.0-2","0.8.0-1","0.6.5-1"]},"yajl":{"dependency_names":["yajl"],"versions":["2.1.0-5","2.1.0-4","2.1.0-3","2.1.0-2","2.1.0-1"]},"yaml-cpp":{"dependency_names":["yaml-cpp"],"versions":["0.8.0-2","0.8.0-1"]},"yyjson":{"dependency_names":["yyjson"],"versions":["0.12.0-1"]},"zlib":{"dependency_names":["zlib"],"versions":["1.3.2-1","1.3.1-3","1.3.1-2","1.3.1-1","1.3-5","1.3-4","1.3-3","1.3-2","1.3-1","1.2.13-4","1.2.13-3","1.2.13-2","1.2.13-1","1.2.12-1","1.2.11-6","1.2.11-5","1.2.11-4","1.2.11-3","1.2.11-2","1.2.11-1","1.2.8-8","1.2.8-7","1.2.8-6","1.2.8-4","1.2.8-2"]},"zlib-ng":{"dependency_names":["zlib-ng"],"versions":["2.3.3-1","2.3.2-2","2.3.2-1","2.3.1-1","2.2.5-1","2.2.4-2","2.2.4-1","2.2.3-1","2.2.2-1"]},"zpp_bits":{"dependency_names":["zpp_bits"],"versions":["4.5.1-1","4.5-1","4.4.24-1","4.4.23-1"]},"zstd":{"dependency_names":["libzstd"],"versions":["1.5.7-3","1.5.7-2","1.5.7-1","1.5.6-2","1.5.6-1","1.5.5-1","1.5.4-1","1.4.5-1","1.3.3-2","1.3.3-1"]},"zycore":{"dependency_names":["zycore"],"versions":["1.5.2-1","1.5.1-1"]},"zydis":{"dependency_names":["zydis"],"versions":["4.1.1-1"]}} diff --git a/test/meson.build b/test/meson.build index 61af0eba..48e69515 100644 --- a/test/meson.build +++ b/test/meson.build @@ -71,6 +71,9 @@ unit_src = [ 'unit/mount.cpp', 'unit/oci_auth.cpp', 'unit/oci_client.cpp', + 'unit/oci_digest.cpp', + 'unit/oci_manifest.cpp', + 'unit/oci_parse.cpp', 'unit/parse.cpp', 'unit/shell.cpp', 'unit/signal.cpp', diff --git a/test/unit/lex.cpp b/test/unit/lex.cpp index 87c94438..fa7d01fa 100644 --- a/test/unit/lex.cpp +++ b/test/unit/lex.cpp @@ -4,7 +4,7 @@ #include TEST_CASE("error characters", "[lex]") { - for (auto in : {"\\", "~", "'", "\""}) { + for (auto in : {"\\", "{", "}", "|", "^", "<", ">", "`"}) { lex::lexer L(in); auto t = L.peek(); REQUIRE(t.kind == lex::tok::error); @@ -12,6 +12,35 @@ TEST_CASE("error characters", "[lex]") { } } +TEST_CASE("url and quote punctuation", "[lex]") { + lex::lexer L("?&~[]$;()'\""); + REQUIRE(L.next() == lex::token{0, lex::tok::question, "?"}); + REQUIRE(L.next() == lex::token{1, lex::tok::amp, "&"}); + REQUIRE(L.next() == lex::token{2, lex::tok::tilde, "~"}); + REQUIRE(L.next() == lex::token{3, lex::tok::lbracket, "["}); + REQUIRE(L.next() == lex::token{4, lex::tok::rbracket, "]"}); + REQUIRE(L.next() == lex::token{5, lex::tok::dollar, "$"}); + REQUIRE(L.next() == lex::token{6, lex::tok::semicolon, ";"}); + REQUIRE(L.next() == lex::token{7, lex::tok::lparen, "("}); + REQUIRE(L.next() == lex::token{8, lex::tok::rparen, ")"}); + REQUIRE(L.next() == lex::token{9, lex::tok::squote, "'"}); + REQUIRE(L.next() == lex::token{10, lex::tok::dquote, "\""}); + REQUIRE(L.next() == lex::token{11, lex::tok::end, ""}); +} + +TEST_CASE("seek", "[lex]") { + lex::lexer L("abc?def"); + REQUIRE(L.peek() == lex::token{0, lex::tok::symbol, "abc"}); + // seek onto the 'def' symbol (offset 4) + L.seek(4); + REQUIRE(L.peek() == lex::token{4, lex::tok::symbol, "def"}); + REQUIRE(L.next() == lex::token{4, lex::tok::symbol, "def"}); + REQUIRE(L.current_kind() == lex::tok::end); + // seeking to the end yields the end token + L.seek(7); + REQUIRE(L.current_kind() == lex::tok::end); +} + TEST_CASE("punctuation", "[lex]") { lex::lexer L(":,:/@!*!#="); REQUIRE(L.next() == lex::token{0, lex::tok::colon, ":"}); diff --git a/test/unit/oci_auth.cpp b/test/unit/oci_auth.cpp index 67b76343..752957a6 100644 --- a/test/unit/oci_auth.cpp +++ b/test/unit/oci_auth.cpp @@ -8,69 +8,19 @@ #include // the pure helpers are not part of the public auth.h interface; redeclare their -// prototypes here (they live in namespace oci::impl in auth.cpp). +// prototypes here (they live in namespace oci::impl in auth.cpp). the +// bearer-challenge parser moved to src/oci/parse.cpp and is covered by +// test/unit/oci_parse.cpp. namespace oci::impl { -std::optional parse_bearer_challenge(std::string_view); std::string token_url(const bearer_challenge&, const std::vector&); std::optional parse_token_response(std::string_view); std::string repository_scope(std::string_view, std::string_view); } // namespace oci::impl -using oci::impl::parse_bearer_challenge; using oci::impl::parse_token_response; using oci::impl::repository_scope; using oci::impl::token_url; -TEST_CASE("oci parse_bearer_challenge from jfrog", "[oci][auth]") { - // the exact challenge the CSCS jfrog registry returns (spike step 1) - auto c = parse_bearer_challenge( - "Bearer realm=\"https://jfrog.svc.cscs.ch/v2/token\"," - "service=\"jfrog.svc.cscs.ch\""); - REQUIRE(c.has_value()); - REQUIRE(c->realm == "https://jfrog.svc.cscs.ch/v2/token"); - REQUIRE(c->service == "jfrog.svc.cscs.ch"); - REQUIRE(c->scopes.empty()); -} - -TEST_CASE("oci parse_bearer_challenge with scope containing a comma", - "[oci][auth]") { - // the scope value itself contains a comma (pull,push) inside quotes - a - // naive comma split would break here. - auto c = parse_bearer_challenge( - "Bearer realm=\"https://r/token\",service=\"reg\"," - "scope=\"repository:foo/bar:pull,push\""); - REQUIRE(c.has_value()); - REQUIRE(c->realm == "https://r/token"); - REQUIRE(c->service == "reg"); - REQUIRE(c->scopes == std::vector{"repository:foo/bar:pull,push"}); -} - -TEST_CASE("oci parse_bearer_challenge space-separated scopes", "[oci][auth]") { - auto c = parse_bearer_challenge( - "Bearer realm=\"https://r/token\",service=\"reg\"," - "scope=\"repository:a:pull repository:b:pull\""); - REQUIRE(c.has_value()); - REQUIRE(c->scopes == std::vector{"repository:a:pull", - "repository:b:pull"}); -} - -TEST_CASE("oci parse_bearer_challenge rejects non-bearer / no realm", - "[oci][auth]") { - REQUIRE_FALSE(parse_bearer_challenge("Basic realm=\"r\"").has_value()); - REQUIRE_FALSE(parse_bearer_challenge("Bearerish realm=\"r\"").has_value()); - REQUIRE_FALSE(parse_bearer_challenge("").has_value()); - // a Bearer challenge without a realm is unusable - REQUIRE_FALSE( - parse_bearer_challenge("Bearer service=\"reg\"").has_value()); -} - -TEST_CASE("oci parse_bearer_challenge is scheme-case-insensitive", - "[oci][auth]") { - auto c = parse_bearer_challenge("bearer realm=\"https://r/token\""); - REQUIRE(c.has_value()); - REQUIRE(c->realm == "https://r/token"); -} - TEST_CASE("oci token_url", "[oci][auth]") { oci::bearer_challenge c{ .realm = "https://jfrog.svc.cscs.ch/v2/token", diff --git a/test/unit/oci_client.cpp b/test/unit/oci_client.cpp index 6a12aed8..bfd77442 100644 --- a/test/unit/oci_client.cpp +++ b/test/unit/oci_client.cpp @@ -1,12 +1,14 @@ #include +#include + #include +#include #include // the pure helpers are not part of the public client.h interface; redeclare // their prototypes here (they live in namespace oci::impl in client.cpp). namespace oci::impl { -std::string digest_string(const util::sha256_digest&); std::string blob_path(std::string_view, std::string_view); std::string manifest_path(std::string_view, std::string_view); std::string uploads_path(std::string_view); @@ -21,13 +23,6 @@ std::optional> parse_referrers(std::string_view); using namespace oci; using namespace oci::impl; -TEST_CASE("oci digest_string", "[oci][client]") { - auto d = util::sha256_string("abc"); - REQUIRE(digest_string(d) == - "sha256:" - "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); -} - TEST_CASE("oci path builders", "[oci][client]") { const std::string repo = "deploy/todi/gh200/app/1.0"; REQUIRE(blob_path(repo, "sha256:abc") == @@ -80,25 +75,34 @@ TEST_CASE("oci parse_tags_list", "[oci][client]") { TEST_CASE("oci parse_referrers", "[oci][client]") { // an image-index body, as returned by /v2//referrers/ - const auto body = R"({ + const auto digest_hex = + "f7f04f3b2cf562336c73542f0c53503c3b853ac459f081878843f878955cf267"; + const auto body = fmt::format(R"({{ "schemaVersion": 2, "mediaType": "application/vnd.oci.image.index.v1+json", "manifests": [ - { + {{ "mediaType": "application/vnd.oci.image.manifest.v1+json", - "digest": "sha256:f7f04f3b", + "digest": "sha256:{}", "size": 1234, "artifactType": "application/vnd.cscs.uenv.meta" - } + }} ] - })"; + }})", + digest_hex); auto refs = parse_referrers(body); REQUIRE(refs.has_value()); REQUIRE(refs->size() == 1); - REQUIRE((*refs)[0].digest == "sha256:f7f04f3b"); + REQUIRE((*refs)[0].digest == digest::sha256(digest_hex)); REQUIRE((*refs)[0].size == 1234); REQUIRE((*refs)[0].artifact_type == "application/vnd.cscs.uenv.meta"); + // an entry with a malformed digest is skipped + auto skipped = parse_referrers( + R"({"manifests":[{"digest":"sha256:short","size":1}]})"); + REQUIRE(skipped.has_value()); + REQUIRE(skipped->empty()); + // no manifests array -> empty list auto empty = parse_referrers(R"({"manifests":[]})"); REQUIRE(empty.has_value()); diff --git a/test/unit/oci_digest.cpp b/test/unit/oci_digest.cpp new file mode 100644 index 00000000..911ae58b --- /dev/null +++ b/test/unit/oci_digest.cpp @@ -0,0 +1,87 @@ +#include + +#include +#include +#include + +using namespace oci; + +TEST_CASE("digest parse valid", "[oci][digest]") { + const std::string hex(64, 'a'); + auto d = digest::parse("sha256:" + hex); + REQUIRE(d.has_value()); + REQUIRE(d->algorithm() == "sha256"); + REQUIRE(d->hex() == hex); + REQUIRE(d->string() == "sha256:" + hex); +} + +TEST_CASE("digest parse rejects malformed", "[oci][digest]") { + // missing separator + REQUIRE_FALSE(digest::parse(std::string(64, 'a')).has_value()); + // unknown algorithm + REQUIRE_FALSE(digest::parse("md5:" + std::string(32, 'a')).has_value()); + // wrong length for sha256 + REQUIRE_FALSE(digest::parse("sha256:abcd").has_value()); + // uppercase is not lowercase hex + REQUIRE_FALSE(digest::parse("sha256:" + std::string(64, 'A')).has_value()); + // non-hex character + REQUIRE_FALSE( + digest::parse("sha256:" + std::string(63, 'a') + "g").has_value()); + // sha512 length is accepted + REQUIRE(digest::parse("sha512:" + std::string(128, 'f')).has_value()); +} + +TEST_CASE("digest from_sha256 and sha256", "[oci][digest]") { + auto h = util::sha256_string("abc"); + auto d = digest::from_sha256(h); + REQUIRE(d.string() == + "sha256:" + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + // the trusted constructor round-trips the same value. + REQUIRE(digest::sha256(h.hex()) == d); +} + +TEST_CASE("digest equality", "[oci][digest]") { + auto a = digest::parse("sha256:" + std::string(64, 'a')).value(); + auto b = digest::parse("sha256:" + std::string(64, 'a')).value(); + auto c = digest::parse("sha256:" + std::string(64, 'b')).value(); + REQUIRE(a == b); + REQUIRE_FALSE(a == c); +} + +TEST_CASE("reference tag vs digest", "[oci][reference]") { + auto t = reference::tag("v3"); + REQUIRE(t.is_tag()); + REQUIRE_FALSE(t.is_digest()); + REQUIRE(t.string() == "v3"); + REQUIRE_FALSE(t.as_digest().has_value()); + + auto d = digest::parse("sha256:" + std::string(64, 'a')).value(); + auto r = reference::digest(d); + REQUIRE(r.is_digest()); + REQUIRE(r.string() == d.string()); + REQUIRE(r.as_digest() == d); +} + +TEST_CASE("reference parse", "[oci][reference]") { + // a digest string parses to a digest reference + auto r = reference::parse("sha256:" + std::string(64, 'a')); + REQUIRE(r.has_value()); + REQUIRE(r->is_digest()); + + // a plain tag parses to a tag reference + auto t = reference::parse("v3"); + REQUIRE(t.has_value()); + REQUIRE(t->is_tag()); + REQUIRE(t->string() == "v3"); + + // the referrers-tag schema (sha256-) is a valid tag + auto rt = reference::parse("sha256-" + std::string(64, 'a')); + REQUIRE(rt.has_value()); + REQUIRE(rt->is_tag()); + + // a bad tag (leading '.') is rejected + REQUIRE_FALSE(reference::parse(".bad").has_value()); + // empty is rejected + REQUIRE_FALSE(reference::parse("").has_value()); +} diff --git a/test/unit/oci_manifest.cpp b/test/unit/oci_manifest.cpp new file mode 100644 index 00000000..2bd0f51f --- /dev/null +++ b/test/unit/oci_manifest.cpp @@ -0,0 +1,178 @@ +#include + +#include +#include +#include +#include + +using namespace oci; + +namespace { +std::string digest_of(const std::string& s) { + return digest::from_sha256(util::sha256_string(s)).string(); +} +const std::string image_layer_hex = + "b563b338a3797eb908b1f60a3b710b7021599aaac61d02f11b273bd2ff0986d4"; +const std::string image_manifest_hex = + "b50ca0d101456970ea94dccb5a33adcfd2b9a566bc36c716f4b64a0522089f48"; +} // namespace + +// The serializer must reproduce, byte-for-byte, the image manifest that +// `oras push --artifact-type application/x-squashfs` produced for the public +// prgenv-gnu/24.7:v3 image. Byte identity means the manifest hashes to the same +// digest oras/JFrog recorded, so images we push co-exist with the old tool. +TEST_CASE("serialize squashfs image manifest is oras-identical", + "[oci][manifest]") { + manifest m; + m.artifact_type = std::string{artifact_type_squashfs}; + m.annotations[std::string{annotation_created}] = "2024-08-23T16:00:40Z"; + m.layers.push_back(manifest_layer{ + .media_type = std::string{media_type_layer_tar}, + .digest = digest::sha256(image_layer_hex), + .size = 4046512128, + .annotations = {{std::string{annotation_title}, "store.squashfs"}}}); + + const std::string expected = + R"({"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","artifactType":"application/x-squashfs","config":{"mediaType":"application/vnd.oci.empty.v1+json","digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","size":2,"data":"e30="},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar","digest":"sha256:b563b338a3797eb908b1f60a3b710b7021599aaac61d02f11b273bd2ff0986d4","size":4046512128,"annotations":{"org.opencontainers.image.title":"store.squashfs"}}],"annotations":{"org.opencontainers.image.created":"2024-08-23T16:00:40Z"}})"; + + auto body = serialize_manifest(m); + REQUIRE(body == expected); + // the recorded manifest digest for this image. + REQUIRE(digest_of(body) == "sha256:" + image_manifest_hex); +} + +// Likewise for the `oras attach --artifact-type uenv/meta` referrer manifest. +TEST_CASE("serialize meta referrer manifest is oras-identical", + "[oci][manifest]") { + manifest m; + m.artifact_type = std::string{artifact_type_meta}; + m.annotations[std::string{annotation_created}] = "2024-08-23T16:01:09Z"; + m.layers.push_back(manifest_layer{ + .media_type = std::string{media_type_layer_targz}, + .digest = digest::sha256( + "3e2e44102d0c54bc2b72295b470b994f128a89b1436d567d850dbf131fcc02db"), + .size = 2366, + .annotations = { + {std::string{annotation_content_digest}, + "sha256:" + "b751bb420ec887b245ea91483849775a8904975aab23b999473b9ea07df9571f"}, + {std::string{annotation_unpack}, "true"}, + {std::string{annotation_title}, "meta"}}}); + m.subject = descriptor{.media_type = std::string{media_type_manifest}, + .digest = digest::sha256(image_manifest_hex), + .size = 588}; + + const std::string expected = + R"({"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","artifactType":"uenv/meta","config":{"mediaType":"application/vnd.oci.empty.v1+json","digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","size":2,"data":"e30="},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:3e2e44102d0c54bc2b72295b470b994f128a89b1436d567d850dbf131fcc02db","size":2366,"annotations":{"io.deis.oras.content.digest":"sha256:b751bb420ec887b245ea91483849775a8904975aab23b999473b9ea07df9571f","io.deis.oras.content.unpack":"true","org.opencontainers.image.title":"meta"}}],"subject":{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:b50ca0d101456970ea94dccb5a33adcfd2b9a566bc36c716f4b64a0522089f48","size":588},"annotations":{"org.opencontainers.image.created":"2024-08-23T16:01:09Z"}})"; + + auto body = serialize_manifest(m); + REQUIRE(body == expected); + REQUIRE(digest_of(body) == + "sha256:" + "f7f04f3b2cf562336c73542f0c53503c3b853ac459f081878843f878955cf267"); +} + +// parse_manifest is the read counterpart: parsing a serialized manifest must +// recover the same structure (round-trip). +TEST_CASE("parse_manifest round-trips serialize_manifest", "[oci][manifest]") { + manifest m; + m.artifact_type = std::string{artifact_type_squashfs}; + m.annotations[std::string{annotation_created}] = "2024-08-23T16:00:40Z"; + m.layers.push_back(manifest_layer{ + .media_type = std::string{media_type_layer_tar}, + .digest = digest::sha256(image_layer_hex), + .size = 4046512128, + .annotations = {{std::string{annotation_title}, "store.squashfs"}}}); + + auto parsed = parse_manifest(serialize_manifest(m)); + REQUIRE(parsed.has_value()); + REQUIRE(parsed->artifact_type == artifact_type_squashfs); + REQUIRE(parsed->config.digest == empty_config_descriptor().digest); + REQUIRE(parsed->config.data == "e30="); + REQUIRE(parsed->layers.size() == 1); + + const auto* layer = parsed->find_layer_by_title("store.squashfs"); + REQUIRE(layer != nullptr); + REQUIRE(layer->digest == digest::sha256(image_layer_hex)); + REQUIRE(layer->size == 4046512128); + REQUIRE_FALSE(parsed->subject.has_value()); + REQUIRE(parsed->annotations.at(std::string{annotation_created}) == + "2024-08-23T16:00:40Z"); +} + +TEST_CASE("parse_manifest finds the unpack layer", "[oci][manifest]") { + manifest m; + m.artifact_type = std::string{artifact_type_meta}; + m.layers.push_back(manifest_layer{ + .media_type = std::string{media_type_layer_targz}, + .digest = digest::sha256(std::string(64, 'a')), + .size = 10, + .annotations = {{std::string{annotation_unpack}, "true"}, + {std::string{annotation_title}, "meta"}}}); + + auto parsed = parse_manifest(serialize_manifest(m)); + REQUIRE(parsed.has_value()); + const auto* layer = parsed->find_unpack_layer(); + REQUIRE(layer != nullptr); + REQUIRE(layer->wants_unpack()); + REQUIRE(layer->title() == "meta"); +} + +TEST_CASE("parse_manifest rejects malformed", "[oci][manifest]") { + REQUIRE_FALSE(parse_manifest("not json").has_value()); + // a layer with an invalid digest is a hard error + REQUIRE_FALSE(parse_manifest( + R"({"layers":[{"digest":"sha256:short"}]})") + .has_value()); +} + +TEST_CASE("serialize image index", "[oci][manifest]") { + descriptor d{.media_type = std::string{media_type_manifest}, + .digest = digest::sha256(image_manifest_hex), + .size = 42, + .artifact_type = "uenv/meta"}; + + const std::string expected = + R"({"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:b50ca0d101456970ea94dccb5a33adcfd2b9a566bc36c716f4b64a0522089f48","size":42,"artifactType":"uenv/meta"}]})"; + REQUIRE(serialize_index({d}) == expected); +} + +TEST_CASE("split_registry", "[oci][manifest]") { + // host with a prefix, no scheme + auto a = split_registry("jfrog.svc.cscs.ch/uenv"); + REQUIRE(a.has_value()); + REQUIRE(a->base == "https://jfrog.svc.cscs.ch"); + REQUIRE(a->prefix == "uenv"); + + // scheme is dropped, https forced + auto b = split_registry("https://jfrog.svc.cscs.ch/uenv"); + REQUIRE(b.has_value()); + REQUIRE(b->base == "https://jfrog.svc.cscs.ch"); + REQUIRE(b->prefix == "uenv"); + + // bare host, no prefix + auto c = split_registry("registry.example"); + REQUIRE(c.has_value()); + REQUIRE(c->base == "https://registry.example"); + REQUIRE(c->prefix == ""); + + // multi-segment prefix, trailing slash trimmed + auto d = split_registry("host.io/a/b/"); + REQUIRE(d.has_value()); + REQUIRE(d->base == "https://host.io"); + REQUIRE(d->prefix == "a/b"); + + // invalid: empty host / whitespace + REQUIRE_FALSE(split_registry("").has_value()); + REQUIRE_FALSE(split_registry("/uenv").has_value()); + REQUIRE_FALSE(split_registry("bad host/uenv").has_value()); +} + +TEST_CASE("repository_path", "[oci][manifest]") { + REQUIRE(repository_path("uenv", "deploy", "daint", "gh200", "prgenv-gnu", + "24.7") == + "uenv/deploy/daint/gh200/prgenv-gnu/24.7"); + // empty prefix omits the leading segment + REQUIRE(repository_path("", "deploy", "daint", "gh200", "prgenv-gnu", + "24.7") == "deploy/daint/gh200/prgenv-gnu/24.7"); +} diff --git a/test/unit/oci_parse.cpp b/test/unit/oci_parse.cpp new file mode 100644 index 00000000..ce9b0077 --- /dev/null +++ b/test/unit/oci_parse.cpp @@ -0,0 +1,189 @@ +#include +#include + +#include + +#include + +using namespace oci; + +TEST_CASE("oci parse_digest valid", "[oci][parse]") { + const std::string hex(64, 'a'); + auto d = parse_digest("sha256:" + hex); + REQUIRE(d.has_value()); + REQUIRE(d->algorithm() == "sha256"); + REQUIRE(d->hex() == hex); + REQUIRE(d->string() == "sha256:" + hex); + + // sha512 length is accepted + REQUIRE(parse_digest("sha512:" + std::string(128, 'f')).has_value()); + // surrounding whitespace is stripped + REQUIRE(parse_digest(" sha256:" + hex + " ").has_value()); + // a value made of only digits is valid hex + REQUIRE(parse_digest("sha256:" + std::string(64, '0')).has_value()); +} + +TEST_CASE("oci parse_digest rejects malformed", "[oci][parse]") { + REQUIRE_FALSE(parse_digest(std::string(64, 'a')).has_value()); // no ':' + REQUIRE_FALSE(parse_digest("md5:" + std::string(32, 'a')).has_value()); + REQUIRE_FALSE(parse_digest("sha256:abcd").has_value()); // too short + REQUIRE_FALSE( + parse_digest("sha256:" + std::string(64, 'A')).has_value()); // upper + REQUIRE_FALSE( + parse_digest("sha256:" + std::string(63, 'a') + "g").has_value()); + REQUIRE_FALSE(parse_digest("sha256:").has_value()); // no value + // trailing junk after a complete digest + REQUIRE_FALSE( + parse_digest("sha256:" + std::string(64, 'a') + ":x").has_value()); +} + +TEST_CASE("oci parse_digest error message has a caret", "[oci][parse]") { + auto d = parse_digest("md5:" + std::string(32, 'a')); + REQUIRE_FALSE(d.has_value()); + // message() renders the input plus a caret underline. + const auto msg = d.error().message(); + REQUIRE(msg.find("md5") != std::string::npos); + REQUIRE(msg.find('^') != std::string::npos); +} + +TEST_CASE("oci parse_reference", "[oci][parse]") { + // a digest string parses to a digest reference + auto r = parse_reference("sha256:" + std::string(64, 'a')); + REQUIRE(r.has_value()); + REQUIRE(r->is_digest()); + + // a plain tag parses to a tag reference + auto t = parse_reference("v3"); + REQUIRE(t.has_value()); + REQUIRE(t->is_tag()); + REQUIRE(t->string() == "v3"); + + // the referrers-tag schema (sha256-) is a valid tag, not a digest + auto rt = parse_reference("sha256-" + std::string(64, 'a')); + REQUIRE(rt.has_value()); + REQUIRE(rt->is_tag()); + + // dotted / underscored tags are valid + REQUIRE(parse_reference("24.7_v2").has_value()); + + // a leading '.' or '-' is rejected, as is empty and an over-long tag + REQUIRE_FALSE(parse_reference(".bad").has_value()); + REQUIRE_FALSE(parse_reference("-bad").has_value()); + REQUIRE_FALSE(parse_reference("").has_value()); + REQUIRE_FALSE(parse_reference(std::string(129, 'a')).has_value()); +} + +TEST_CASE("oci parse_url components", "[oci][parse]") { + // scheme-less host with a path + auto a = parse_url("jfrog.svc.cscs.ch/uenv"); + REQUIRE(a.has_value()); + REQUIRE(a->scheme == ""); + REQUIRE(a->host == "jfrog.svc.cscs.ch"); + REQUIRE(a->path == "/uenv"); + REQUIRE_FALSE(a->port.has_value()); + + // scheme is captured, path preserved + auto b = parse_url("https://host.io/a/b/"); + REQUIRE(b.has_value()); + REQUIRE(b->scheme == "https"); + REQUIRE(b->host == "host.io"); + REQUIRE(b->path == "/a/b/"); + + // bare host, no path + auto c = parse_url("registry.example"); + REQUIRE(c.has_value()); + REQUIRE(c->host == "registry.example"); + REQUIRE(c->path == ""); + + // port, query and fragment + auto d = parse_url("http://h:8080/p?a=1&b=2#frag"); + REQUIRE(d.has_value()); + REQUIRE(d->scheme == "http"); + REQUIRE(d->host == "h"); + REQUIRE(d->port == 8080u); + REQUIRE(d->path == "/p"); + REQUIRE(d->query == "a=1&b=2"); + REQUIRE(d->fragment == "frag"); + + // IPv6 literal host with a port + auto e = parse_url("https://[::1]:5000/x"); + REQUIRE(e.has_value()); + REQUIRE(e->host == "[::1]"); + REQUIRE(e->port == 5000u); + REQUIRE(e->path == "/x"); + + // userinfo + auto f = parse_url("https://user:pass@host/p"); + REQUIRE(f.has_value()); + REQUIRE(f->userinfo == "user:pass"); + REQUIRE(f->host == "host"); + + // round-trips through string() + REQUIRE(parse_url("https://h:8080/p?q=1#f")->string() == + "https://h:8080/p?q=1#f"); +} + +TEST_CASE("oci parse_url rejects malformed", "[oci][parse]") { + REQUIRE_FALSE(parse_url("").has_value()); + REQUIRE_FALSE(parse_url("/uenv").has_value()); // no host + REQUIRE_FALSE(parse_url("bad host/uenv").has_value()); // whitespace in host + REQUIRE_FALSE(parse_url("host:notaport/x").has_value()); +} + +TEST_CASE("oci parse_bearer_challenge from jfrog", "[oci][parse]") { + auto c = parse_bearer_challenge( + "Bearer realm=\"https://jfrog.svc.cscs.ch/v2/token\"," + "service=\"jfrog.svc.cscs.ch\""); + REQUIRE(c.has_value()); + REQUIRE(c->realm == "https://jfrog.svc.cscs.ch/v2/token"); + REQUIRE(c->service == "jfrog.svc.cscs.ch"); + REQUIRE(c->scopes.empty()); +} + +TEST_CASE("oci parse_bearer_challenge with a scope containing a comma", + "[oci][parse]") { + // the quoted scope value itself contains a comma (pull,push); a naive comma + // split would break here. + auto c = parse_bearer_challenge( + "Bearer realm=\"https://r/token\",service=\"reg\"," + "scope=\"repository:foo/bar:pull,push\""); + REQUIRE(c.has_value()); + REQUIRE(c->realm == "https://r/token"); + REQUIRE(c->service == "reg"); + REQUIRE(c->scopes == + std::vector{"repository:foo/bar:pull,push"}); +} + +TEST_CASE("oci parse_bearer_challenge space-separated scopes", "[oci][parse]") { + auto c = parse_bearer_challenge( + "Bearer realm=\"https://r/token\",service=\"reg\"," + "scope=\"repository:a:pull repository:b:pull\""); + REQUIRE(c.has_value()); + REQUIRE(c->scopes == std::vector{"repository:a:pull", + "repository:b:pull"}); +} + +TEST_CASE("oci parse_bearer_challenge rejects non-bearer / no realm", + "[oci][parse]") { + REQUIRE_FALSE(parse_bearer_challenge("Basic realm=\"r\"").has_value()); + REQUIRE_FALSE(parse_bearer_challenge("Bearerish realm=\"r\"").has_value()); + REQUIRE_FALSE(parse_bearer_challenge("").has_value()); + REQUIRE_FALSE(parse_bearer_challenge("Bearer service=\"reg\"").has_value()); +} + +TEST_CASE("oci parse_bearer_challenge is scheme-case-insensitive", + "[oci][parse]") { + auto c = parse_bearer_challenge("bearer realm=\"https://r/token\""); + REQUIRE(c.has_value()); + REQUIRE(c->realm == "https://r/token"); +} + +TEST_CASE("oci parse_scopes", "[oci][parse]") { + REQUIRE(parse_scopes("").empty()); + REQUIRE(parse_scopes(" ").empty()); + REQUIRE(parse_scopes("repository:a:pull") == + std::vector{"repository:a:pull"}); + // leading/trailing/multiple spaces are ignored + REQUIRE(parse_scopes(" a b\tc ") == + std::vector{"a", "b", "c"}); +} From 84b95ee0ff0373f2eab19a28a6a1f115e266fcb2 Mon Sep 17 00:00:00 2001 From: bcumming Date: Sat, 4 Jul 2026 19:26:23 +0200 Subject: [PATCH 09/29] add registry testing --- src/cli/copy.cpp | 6 +- src/cli/delete.cpp | 2 +- src/cli/find.cpp | 2 +- src/cli/pull.cpp | 2 +- src/cli/push.cpp | 2 +- src/oci/auth.cpp | 15 +- src/oci/auth.h | 7 +- src/oci/client.cpp | 16 ++- src/oci/client.h | 7 +- src/site/site.cpp | 15 +- src/site/site.h | 8 +- src/uenv/settings.cpp | 11 +- src/uenv/settings.h | 4 + test/integration/cli.bats | 12 ++ test/integration/common.bash | 37 +++++ test/integration/install-zot | 35 +++++ test/integration/listing_mock | 193 ++++++++++++++++++++++++++ test/integration/readme.md | 7 +- test/integration/registry.bats | 108 +++++++++++++++ test/integration/registry_ctl | 162 ++++++++++++++++++++++ test/integration/slurm.bats | 21 ++- test/meson.build | 32 +++++ test/unit/oci_manifest.cpp | 7 + test/unit/oci_registry.cpp | 245 +++++++++++++++++++++++++++++++++ test/unit/settings.cpp | 27 ++++ 25 files changed, 950 insertions(+), 33 deletions(-) create mode 100755 test/integration/install-zot create mode 100755 test/integration/listing_mock create mode 100644 test/integration/registry.bats create mode 100755 test/integration/registry_ctl create mode 100644 test/unit/oci_registry.cpp diff --git a/src/cli/copy.cpp b/src/cli/copy.cpp index b867c67d..5b45b2dc 100644 --- a/src/cli/copy.cpp +++ b/src/cli/copy.cpp @@ -100,7 +100,8 @@ int image_copy([[maybe_unused]] const image_copy_args& args, return 1; } - auto src_registry = site::registry_listing(*src_label.nspace); + auto src_registry = + site::registry_listing(registry_cfg.listing_url, *src_label.nspace); if (!src_registry) { term::error("unable to get a listing of the uenv", src_registry.error()); @@ -163,7 +164,8 @@ int image_copy([[maybe_unused]] const image_copy_args& args, spdlog::info("destination record: {} {}", dst_record.sha, dst_record); // check whether the destination already exists - auto dst_registry = site::registry_listing(*dst_label.nspace); + auto dst_registry = + site::registry_listing(registry_cfg.listing_url, *dst_label.nspace); if (dst_registry && dst_registry->contains(dst_record)) { if (!args.force) { term::error("the destination already exists - use the --force flag " diff --git a/src/cli/delete.cpp b/src/cli/delete.cpp index 40e45718..37c54472 100644 --- a/src/cli/delete.cpp +++ b/src/cli/delete.cpp @@ -91,7 +91,7 @@ int image_delete([[maybe_unused]] const image_delete_args& args, } spdlog::debug("requested to delete {}::{}", nspace, label); - auto registry = site::registry_listing(nspace); + auto registry = site::registry_listing(registry_cfg.listing_url, nspace); if (!registry) { term::error("unable to get a listing of the uenv", registry.error()); return 1; diff --git a/src/cli/find.cpp b/src/cli/find.cpp index 93972496..0f15b77e 100644 --- a/src/cli/find.cpp +++ b/src/cli/find.cpp @@ -85,7 +85,7 @@ int image_find([[maybe_unused]] const image_find_args& args, label = apply_system(label, settings.config.system_name); spdlog::info("image_find: {}::{}", nspace, label); - auto store = site::registry_listing(nspace); + auto store = site::registry_listing(registry_cfg.listing_url, nspace); if (!store) { term::error("unable to get a listing of the uenv", store.error()); return 1; diff --git a/src/cli/pull.cpp b/src/cli/pull.cpp index 5b3b88f1..77393eec 100644 --- a/src/cli/pull.cpp +++ b/src/cli/pull.cpp @@ -105,7 +105,7 @@ int image_pull(const image_pull_args& args, const global_settings& settings) { spdlog::info("image_pull: {}::{}", nspace, label); - auto registry = site::registry_listing(nspace); + auto registry = site::registry_listing(registry_cfg.listing_url, nspace); if (!registry) { term::error("unable to get a listing of the uenv", registry.error()); return 1; diff --git a/src/cli/push.cpp b/src/cli/push.cpp index c3e6b35e..16beeff8 100644 --- a/src/cli/push.cpp +++ b/src/cli/push.cpp @@ -94,7 +94,7 @@ int image_push([[maybe_unused]] const image_push_args& args, dst_label.label); const auto nspace = dst_label.nspace.value(); - auto registry = site::registry_listing(nspace); + auto registry = site::registry_listing(registry_cfg.listing_url, nspace); if (!registry) { term::error("unable to get a listing of the uenv", registry.error()); return 1; diff --git a/src/oci/auth.cpp b/src/oci/auth.cpp index 5574f436..e9972b38 100644 --- a/src/oci/auth.cpp +++ b/src/oci/auth.cpp @@ -66,7 +66,7 @@ std::string repository_scope(std::string_view repository, } // namespace impl -util::expected +util::expected, std::string> discover_challenge(const std::string& registry_url) { // normalise: drop any trailing '/' before appending the v2 base path. std::string base = registry_url; @@ -84,9 +84,10 @@ discover_challenge(const std::string& registry_url) { resp.error().message)}; } if (resp->status == 200) { - return util::unexpected{fmt::format( - "{} permits anonymous access and issued no auth challenge", - req.url)}; + // the registry permits anonymous access and issued no challenge; the + // client will operate tokenless. + spdlog::trace("{} permits anonymous access", req.url); + return std::nullopt; } if (resp->status != 401) { return util::unexpected{fmt::format( @@ -144,7 +145,11 @@ authenticate(const std::string& registry_url, if (!challenge) { return util::unexpected{challenge.error()}; } - return fetch_token(*challenge, scopes, creds); + // anonymous registry: no token required. + if (!*challenge) { + return std::string{}; + } + return fetch_token(**challenge, scopes, creds); } util::expected, std::string> diff --git a/src/oci/auth.h b/src/oci/auth.h index b2bf3abc..8abb7385 100644 --- a/src/oci/auth.h +++ b/src/oci/auth.h @@ -31,8 +31,11 @@ struct bearer_challenge { // --- network operations ------------------------------------------------- -// Probe `/v2/` and parse the Bearer challenge from the 401. -util::expected +// Probe `/v2/` and parse the auth challenge. Returns the parsed +// Bearer challenge on a 401, or `std::nullopt` when the registry permits +// anonymous access (200 on /v2/, no challenge) — in which case the client +// operates tokenless. +util::expected, std::string> discover_challenge(const std::string& registry_url); // Request a bearer token for the given scopes, optionally authenticating with diff --git a/src/oci/client.cpp b/src/oci/client.cpp index e07a29fb..6c0f74ae 100644 --- a/src/oci/client.cpp +++ b/src/oci/client.cpp @@ -140,7 +140,10 @@ split_registry(std::string_view configured_url) { prefix.pop_back(); } - std::string base = "https://" + u->host; + // preserve an explicit scheme (e.g. http:// for a local test registry); + // default to https when the config omits one (the CSCS deployment). + const std::string scheme = u->scheme.empty() ? "https" : u->scheme; + std::string base = scheme + "://" + u->host; if (u->port) { base += ':' + std::to_string(*u->port); } @@ -179,7 +182,12 @@ client::create(std::string registry_url, std::string repository, return c; } -util::expected client::token_for(bool write) { +util::expected, std::string> +client::token_for(bool write) { + // anonymous registry: no challenge, so no token is needed or fetched. + if (!challenge_) { + return std::nullopt; + } auto& cache = write ? push_token_ : pull_token_; if (cache) { return *cache; @@ -191,7 +199,7 @@ util::expected client::token_for(bool write) { for (const auto& r : extra_pull_scopes_) { scopes.push_back(impl::repository_scope(r, "pull")); } - auto token = fetch_token(challenge_, scopes, creds_); + auto token = fetch_token(*challenge_, scopes, creds_); if (!token) { return util::unexpected{token.error()}; } @@ -365,7 +373,7 @@ namespace { // payload (carried by `body_setup`) to the returned Location with ?digest=. util::expected do_put_blob(const std::string& registry_url, const std::string& uploads, - const std::string& token, const std::string& digest, + const std::optional& token, const std::string& digest, const std::function& body_setup) { // 1. open an upload session util::curl::request post; diff --git a/src/oci/client.h b/src/oci/client.h index 6b3869ad..2111e89f 100644 --- a/src/oci/client.h +++ b/src/oci/client.h @@ -149,12 +149,15 @@ class client { client() = default; // return a bearer token for this repository, fetching/caching as needed. - util::expected token_for(bool write); + // returns std::nullopt for an anonymous registry (no auth challenge), in + // which case requests are sent without an Authorization header. + util::expected, std::string> token_for(bool write); std::string registry_url_; // normalised, no trailing '/' std::string repository_; std::optional creds_; - bearer_challenge challenge_; + // the auth challenge, or nullopt when the registry permits anonymous access. + std::optional challenge_; std::optional pull_token_; std::optional push_token_; // extra repositories to request pull scope for (cross-repo mount). diff --git a/src/site/site.cpp b/src/site/site.cpp index b65bde8c..fb3b0420 100644 --- a/src/site/site.cpp +++ b/src/site/site.cpp @@ -15,24 +15,27 @@ namespace site { util::expected -registry_listing(const std::string& nspace) { +registry_listing(const std::optional& listing_url, + const std::string& nspace) { using json = nlohmann::json; // perform curl call against middleware end point // example of full url end point call: // https://uenv-list.svc.cscs.ch/list?namespace=deploy&cluster=todi&arch=gh200&app=prgenv-gnu&version=24.7 // we only filter on namespace, and use the database to do more querying - // later - const auto url = - fmt::format("https://uenv-list.svc.cscs.ch/list?namespace={}", nspace); + // later. the base URL can be overridden via registry.listing_url (e.g. to a + // local mock endpoint for testing). + const auto base = + listing_url.value_or("https://uenv-list.svc.cscs.ch/list"); + const auto url = fmt::format("{}?namespace={}", base, nspace); spdlog::debug("registry_listing: {}", url); auto raw_records = util::curl::get(url); if (!raw_records) { int ec = raw_records.error().code; spdlog::error("curl error {}: {}", ec, raw_records.error().message); - return util::unexpected{"unable to reach uenv-list.svc.cscs.ch to get " - "list of available uenv"}; + return util::unexpected{fmt::format( + "unable to reach {} to get list of available uenv", base)}; } std::vector records; diff --git a/src/site/site.h b/src/site/site.h index 81aea01a..e90e9887 100644 --- a/src/site/site.h +++ b/src/site/site.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -7,8 +8,13 @@ namespace site { +// query the uenv listing service for the uenv available in a namespace. +// listing_url overrides the service base URL (default: +// https://uenv-list.svc.cscs.ch/list) — used to point at a local/mock endpoint +// for testing. util::expected -registry_listing(const std::string& nspace); +registry_listing(const std::optional& listing_url, + const std::string& nspace); // return the name of the current system from the calling environment. // on CSCS systems this is derived from the CLUSTER_NAME environment variable. diff --git a/src/uenv/settings.cpp b/src/uenv/settings.cpp index c75aa6e4..83c68a77 100644 --- a/src/uenv/settings.cpp +++ b/src/uenv/settings.cpp @@ -585,6 +585,7 @@ parse_registry(const toml::node& input) { std::optional url{}; std::optional default_namespace{}; std::optional artifactory_url{}; + std::optional listing_url{}; for (const auto& entry : *tbl) { std::string_view key = entry.first.str(); @@ -596,6 +597,13 @@ parse_registry(const toml::node& input) { return make_config_error("registry.url must be a string", value.source().begin.line); } + } else if (key == "listing_url") { + if (auto v = value.value()) { + listing_url = v.value(); + } else { + return make_config_error("registry.listing_url must be a string", + value.source().begin.line); + } } else if (key == "default_namespace") { if (auto v = value.value()) { default_namespace = v.value(); @@ -629,7 +637,8 @@ parse_registry(const toml::node& input) { return registry_config{.url = std::move(url.value()), .default_namespace = std::move(default_namespace.value()), - .artifactory_url = std::move(artifactory_url)}; + .artifactory_url = std::move(artifactory_url), + .listing_url = std::move(listing_url)}; } util::expected diff --git a/src/uenv/settings.h b/src/uenv/settings.h index bce11412..c0a3eed9 100644 --- a/src/uenv/settings.h +++ b/src/uenv/settings.h @@ -15,6 +15,10 @@ struct registry_config { std::string url; std::string default_namespace; std::optional artifactory_url; + // overrides the uenv listing service base URL (default: + // https://uenv-list.svc.cscs.ch/list); used to point at a local/mock + // listing endpoint for testing. + std::optional listing_url; }; struct config_base { diff --git a/test/integration/cli.bats b/test/integration/cli.bats index 7325b9bb..dc34a038 100644 --- a/test/integration/cli.bats +++ b/test/integration/cli.bats @@ -19,6 +19,16 @@ function setup() { rm -rf $TMP mkdir -p $TMP + # force the system name to arapiles via a user config file. + # a system config (e.g. /etc/uenv/config.toml on Alps) sets system_name and + # takes priority over the CLUSTER_NAME environment variable, so tests point + # XDG_CONFIG_HOME at a throwaway user config that wins over the system one. + export XDG_CONFIG_HOME=$TMP/user-config + mkdir -p $XDG_CONFIG_HOME/uenv + cat > $XDG_CONFIG_HOME/uenv/config.toml < $XDG/uenv/config.toml </dev/null || true + REGISTRY_STATE="" + fi +} + +# start_listing LISTING_FILE PORT: run the listing_mock server on 127.0.0.1:PORT. +function start_listing() { + local listing_file="$1" + local port="$2" + listing_mock serve "$listing_file" "$port" & + LISTING_MOCK_PID=$! + listing_mock wait-server "$port" --timeout 5 +} + +function stop_listing() { + if [[ -n "${LISTING_MOCK_PID:-}" ]]; then + kill "$LISTING_MOCK_PID" 2>/dev/null || true + wait "$LISTING_MOCK_PID" 2>/dev/null || true + LISTING_MOCK_PID="" + fi +} + function run_srun_unchecked() { log "+ srun $@" run srun -n1 --oversubscribe "$@" diff --git a/test/integration/install-zot b/test/integration/install-zot new file mode 100755 index 00000000..5b8c2ba5 --- /dev/null +++ b/test/integration/install-zot @@ -0,0 +1,35 @@ +#!/bin/bash +# Download a pinned, architecture-matched zot binary for the registry +# integration tests. Invoked by meson at build time. +# +# Usage: install-zot +# is "amd64" or "arm64" (zot release asset suffix). + +set -euo pipefail + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +output=$1 +arch=$2 +version=v2.1.1 + +url="https://github.com/project-zot/zot/releases/download/${version}/zot-linux-${arch}" + +# download to a temp file then move into place, so a failed/partial download +# does not leave a broken "zot" that looks complete to ninja. +tmp="${output}.download" +if curl -fL "$url" -o "$tmp" 2>/dev/null; then + chmod +x "$tmp" + mv "$tmp" "$output" +else + rm -f "$tmp" + # leave no binary behind; the tests self-skip when zot is absent. emit a + # marker file so the custom_target still produces its declared output. + echo "install-zot: failed to download $url (registry tests will skip)" >&2 + rm -f "$output" + # create an empty, non-executable placeholder so meson sees the output exist + : > "$output" +fi diff --git a/test/integration/listing_mock b/test/integration/listing_mock new file mode 100755 index 00000000..55e99412 --- /dev/null +++ b/test/integration/listing_mock @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +"""listing_mock — fake uenv listing service for integration tests. + +Stands in for https://uenv-list.svc.cscs.ch/list so `uenv pull`/`push`/`copy`/ +`find` can resolve labels against a local, controllable catalogue while the +image bytes live in a local zot registry. Point uenv at it with +`registry.listing_url = "http://127.0.0.1:PORT/list"` in the config. + +The service answers `GET /list?namespace=` with the CSCS schema consumed by +site::registry_listing: + + {"results": [{"sha256": ..., "created": ..., "path": ..., "size": ...}, ...]} + +where `path` is "/////". Records are +maintained in a JSON-lines file so a test can push an image, read back its +manifest digest, register a record, and then pull. + +Subcommands +----------- +free-port print a free TCP port and exit +serve LISTING_FILE PORT run HTTP server on 127.0.0.1:PORT serving + /list?namespace=; writes PID to + LISTING_FILE.pid on startup +wait-server PORT [--timeout N] wait until server accepts connections +add LISTING_FILE --path P --sha S append a record (size/created optional; + [--size N] [--created TS] created defaults to a fixed timestamp) +kill LISTING_FILE stop a serve process via its PID file +""" +import argparse +import json +import os +import signal +import socket +import sys +import time +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path +from urllib.parse import urlparse, parse_qs + +DEFAULT_CREATED = "2024-01-01T00:00:00.000Z" + + +def _pid_path(listing_file: str) -> Path: + return Path(listing_file).with_suffix(".pid") + + +def _records(listing_file: str) -> list: + p = Path(listing_file) + if not p.exists(): + return [] + return [json.loads(line) for line in p.read_text().splitlines() if line.strip()] + + +# ── subcommands ─────────────────────────────────────────────────────────────── + +def cmd_free_port(_args): + s = socket.socket() + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind(("", 0)) + port = s.getsockname()[1] + s.close() + print(port) + + +def cmd_serve(args): + listing_file = args.listing_file + pid_path = _pid_path(listing_file) + pid_path.write_text(str(os.getpid())) + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + parsed = urlparse(self.path) + if parsed.path.rstrip("/") not in ("/list", ""): + self.send_response(404) + self.end_headers() + return + nspace = parse_qs(parsed.query).get("namespace", [None])[0] + results = [] + for r in _records(listing_file): + path = r.get("path", "") + if nspace is None or path.split("/")[0] == nspace: + results.append(r) + payload = json.dumps({"results": results}).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *_): + pass + + def cleanup(sig, frame): + pid_path.unlink(missing_ok=True) + sys.exit(0) + + signal.signal(signal.SIGTERM, cleanup) + signal.signal(signal.SIGINT, cleanup) + + try: + HTTPServer(("127.0.0.1", args.port), Handler).serve_forever() + finally: + pid_path.unlink(missing_ok=True) + + +def cmd_wait_server(args): + deadline = time.monotonic() + args.timeout + while time.monotonic() < deadline: + try: + s = socket.socket() + s.settimeout(0.2) + s.connect(("127.0.0.1", args.port)) + s.close() + sys.exit(0) + except OSError: + pass + time.sleep(0.1) + print(f"error: server on port {args.port} did not start within {args.timeout}s", + file=sys.stderr) + sys.exit(1) + + +def cmd_add(args): + # the sha256 is stored bare (no "sha256:" prefix), matching the CSCS schema + # (site.cpp promotes it with uenv::sha256()). + sha = args.sha + if sha.startswith("sha256:"): + sha = sha[len("sha256:"):] + record = { + "sha256": sha, + "created": args.created, + "path": args.path, + "size": args.size, + } + with open(args.listing_file, "a") as f: + f.write(json.dumps(record) + "\n") + + +def cmd_kill(args): + pid_path = _pid_path(args.listing_file) + if not pid_path.exists(): + print(f"error: no PID file found for {args.listing_file}", file=sys.stderr) + sys.exit(1) + pid = int(pid_path.read_text().strip()) + try: + os.kill(pid, signal.SIGTERM) + except ProcessLookupError: + pass + pid_path.unlink(missing_ok=True) + + +# ── argument parsing ────────────────────────────────────────────────────────── + +def main(): + p = argparse.ArgumentParser( + prog="listing_mock", + description="Fake uenv listing service for integration tests.") + sub = p.add_subparsers(dest="cmd", required=True) + + sub.add_parser("free-port", help="print a free TCP port and exit") + + sp = sub.add_parser("serve", help="run the mock listing HTTP server") + sp.add_argument("listing_file", help="JSON-lines file of records to serve") + sp.add_argument("port", type=int) + + sp = sub.add_parser("wait-server", help="wait until server accepts connections") + sp.add_argument("port", type=int) + sp.add_argument("--timeout", type=float, default=5.0, metavar="SECONDS") + + sp = sub.add_parser("add", help="append a listing record") + sp.add_argument("listing_file") + sp.add_argument("--path", required=True, + help="/////") + sp.add_argument("--sha", required=True, help="manifest sha256 (bare hex)") + sp.add_argument("--size", type=int, default=0) + sp.add_argument("--created", default=DEFAULT_CREATED) + + sp = sub.add_parser("kill", help="stop a serve process via its PID file") + sp.add_argument("listing_file") + + args = p.parse_args() + dispatch = { + "free-port": cmd_free_port, + "serve": cmd_serve, + "wait-server": cmd_wait_server, + "add": cmd_add, + "kill": cmd_kill, + } + dispatch[args.cmd](args) + + +if __name__ == "__main__": + main() diff --git a/test/integration/readme.md b/test/integration/readme.md index 329167df..1e7778f9 100644 --- a/test/integration/readme.md +++ b/test/integration/readme.md @@ -14,7 +14,7 @@ This will generate the test inputs in the `test` subdirectory of the build path, ## running the tests -There are three separate sets of tests. +There are four separate sets of tests. - `slurm.bats`: test the slurm plugin - ensure that you have built the plugin and configured slurm in your environment to use the plugin. @@ -22,11 +22,16 @@ There are three separate sets of tests. - ensure that you have built the cli and that it is in your `PATH`. - `squashfs-mount.bats`: test the squashfs-mount cli tool - the tool is setuid, so an additional installation step is required (see below) +- `registry.bats`: test `uenv image push`/`pull` against a throwaway local zot + registry (a binary downloaded by meson; no container runtime needed). Self-skips + if the zot binary is unavailable. The native OCI round-trip is covered by the + `[registry]` unit tests, which start their own zot. To run the respective tests, build with `-Dtests=enabled`, then in the build path: ```console ./test/bats ./test/slurm.bats ./test/bats ./test/cli.bats +./test/bats ./test/registry.bats ``` ## testing squashfs-mount diff --git a/test/integration/registry.bats b/test/integration/registry.bats new file mode 100644 index 00000000..e0c04700 --- /dev/null +++ b/test/integration/registry.bats @@ -0,0 +1,108 @@ +# CLI tests that drive `uenv push`/`pull` against a throwaway zot registry (a +# binary, no container). A listing_mock stands in for the CSCS listing service. +# The suite self-skips when no zot binary is available. +# +# The native OCI round-trip is covered by the [registry] unit cases, which start +# their own zot; this suite covers only the user-facing CLI path. +# +# Run from the build directory: ./test/bats ./test/registry.bats + +function setup_file() { + load ./common + if [[ -z "$(registry_ctl runtime)" ]]; then + return 0 # tests self-skip + fi + + export REG_PORT="$(registry_ctl free-port)" + export REG_STATE="$(mktemp -u)" + export REG_PREFIX="uenvtest" + registry_ctl serve "$REG_STATE" "$REG_PORT" & + registry_ctl wait "$REG_PORT" --timeout 30 + + export LST_PORT="$(listing_mock free-port)" + export LISTING_FILE="$(mktemp)" + listing_mock serve "$LISTING_FILE" "$LST_PORT" & + listing_mock wait-server "$LST_PORT" --timeout 5 + + export REG_CONFIG="$(mktemp -d)" + mkdir -p "$REG_CONFIG/uenv" + cat > "$REG_CONFIG/uenv/config.toml" </dev/null || true + fi + if [[ -n "${LISTING_FILE:-}" ]]; then + listing_mock kill "$LISTING_FILE" 2>/dev/null || true + fi + if [[ -n "${REG_CONFIG:-}" ]]; then + rm -rf "$REG_CONFIG" + fi +} + +function setup() { + set -u + export CLUSTER_NAME=arapiles + + bats_load_library bats-support + bats_load_library bats-assert + load ./common + + export PATH="$UENV_BIN_PATH:$PATH" + unset UENV_MOUNT_LIST + unset -f uenv + + if [[ -z "$(registry_ctl runtime)" ]]; then + skip "no zot binary available" + fi + + export XDG_CONFIG_HOME="$REG_CONFIG" + + export TMP=$DATA/scratch + rm -rf $TMP + mkdir -p $TMP +} + +function teardown() { + : +} + +@test "uenv push then pull round-trips via the registry" { + local sqfs=$SQFS_LIB/apptool/standalone/tool.squashfs + [ -f "$sqfs" ] + + run uenv image push "$sqfs" "deploy::app/1.0:v1@arapiles%zen3" + log "${output}" + [ "${status}" -eq 0 ] + + # register a listing record so pull can resolve the label + local repo="${REG_PREFIX}/deploy/arapiles/zen3/app/1.0" + local digest + digest="$(registry_ctl digest "$REG_PORT" "$repo" v1)" + [ -n "$digest" ] + local size + size="$(stat -c%s "$sqfs")" + listing_mock add "$LISTING_FILE" \ + --path "deploy/arapiles/zen3/app/1.0/v1" --sha "$digest" --size "$size" + + local RP=$TMP/repo + run uenv repo create "$RP" + log "${output}" + [ "${status}" -eq 0 ] + + run uenv --repo "$RP" image pull "deploy::app/1.0:v1@arapiles%zen3" + log "${output}" + [ "${status}" -eq 0 ] + + run uenv --repo "$RP" image ls "app/1.0:v1@arapiles%zen3" + log "${output}" + [ "${status}" -eq 0 ] + assert_output --partial "app" +} diff --git a/test/integration/registry_ctl b/test/integration/registry_ctl new file mode 100755 index 00000000..73365824 --- /dev/null +++ b/test/integration/registry_ctl @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""registry_ctl — run a throwaway zot registry (a binary, no container) for tests. + +Subcommands +----------- +runtime print "zot" if a zot binary is available, else nothing +free-port print a free TCP port +serve STATE PORT run zot on 127.0.0.1:PORT (anonymous, http); state + (config, storage, pidfile) lives under STATE* +wait PORT [--timeout N] wait until GET /v2/ answers (default 30 s) +digest PORT REPO REF print the stored manifest digest (bare hex) +kill STATE stop a serve process via its pidfile +""" +import argparse +import json +import os +import shutil +import socket +import signal +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + + +def _zot_binary() -> str: + # explicit override, then the binary meson downloads next to this script, + # then PATH. + env = os.environ.get("UENV_TEST_ZOT") + if env and os.access(env, os.X_OK): + return env + sibling = Path(__file__).resolve().parent / "zot" + if os.access(sibling, os.X_OK): + return str(sibling) + found = shutil.which("zot") + return found or "" + + +def _pid_path(state: str) -> Path: + return Path(state + ".pid") + + +def cmd_runtime(_args): + print("zot" if _zot_binary() else "") + + +def cmd_free_port(_args): + s = socket.socket() + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind(("", 0)) + print(s.getsockname()[1]) + s.close() + + +def cmd_serve(args): + zot = _zot_binary() + if not zot: + print("error: no zot binary available", file=sys.stderr) + sys.exit(1) + # anonymous + plain-http: no auth/accessControl -> /v2/ returns 200. + state_d = Path(args.state + ".d") + state_d.mkdir(parents=True, exist_ok=True) + cfg = state_d / "config.json" + cfg.write_text(json.dumps({ + "storage": {"rootDirectory": str(state_d / "registry")}, + "http": {"address": "127.0.0.1", "port": str(args.port)}, + "log": {"level": "error", "output": str(state_d / "zot.log")}, + })) + _pid_path(args.state).write_text(str(os.getpid())) + os.execv(zot, [zot, "serve", str(cfg)]) + + +def cmd_wait(args): + url = f"http://127.0.0.1:{args.port}/v2/" + deadline = time.monotonic() + args.timeout + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(url, timeout=1) as resp: + if resp.status == 200: + sys.exit(0) + except urllib.error.HTTPError as e: + if e.code in (200, 401): + sys.exit(0) + except (urllib.error.URLError, OSError): + pass + time.sleep(0.2) + print(f"error: registry on port {args.port} not ready within {args.timeout}s", + file=sys.stderr) + sys.exit(1) + + +MANIFEST_ACCEPT = ("application/vnd.oci.image.manifest.v1+json," + "application/vnd.oci.image.index.v1+json") + + +def cmd_digest(args): + url = f"http://127.0.0.1:{args.port}/v2/{args.repo}/manifests/{args.ref}" + req = urllib.request.Request(url, method="GET", + headers={"Accept": MANIFEST_ACCEPT}) + try: + with urllib.request.urlopen(req, timeout=5) as resp: + dg = resp.headers.get("Docker-Content-Digest", "") + except urllib.error.HTTPError as e: + print(f"error: manifest {args.repo}:{args.ref} not found (status {e.code})", + file=sys.stderr) + sys.exit(1) + if not dg: + print("error: no Docker-Content-Digest header", file=sys.stderr) + sys.exit(1) + if dg.startswith("sha256:"): + dg = dg[len("sha256:"):] + print(dg) + + +def cmd_kill(args): + pid_path = _pid_path(args.state) + if not pid_path.exists(): + return + try: + os.kill(int(pid_path.read_text().strip()), signal.SIGTERM) + except (ProcessLookupError, ValueError): + pass + pid_path.unlink(missing_ok=True) + + +def main(): + p = argparse.ArgumentParser(prog="registry_ctl") + sub = p.add_subparsers(dest="cmd", required=True) + + sub.add_parser("runtime") + sub.add_parser("free-port") + + sp = sub.add_parser("serve") + sp.add_argument("state") + sp.add_argument("port", type=int) + + sp = sub.add_parser("wait") + sp.add_argument("port", type=int) + sp.add_argument("--timeout", type=float, default=30.0) + + sp = sub.add_parser("digest") + sp.add_argument("port", type=int) + sp.add_argument("repo") + sp.add_argument("ref") + + sp = sub.add_parser("kill") + sp.add_argument("state") + + args = p.parse_args() + { + "runtime": cmd_runtime, + "free-port": cmd_free_port, + "serve": cmd_serve, + "wait": cmd_wait, + "digest": cmd_digest, + "kill": cmd_kill, + }[args.cmd](args) + + +if __name__ == "__main__": + main() diff --git a/test/integration/slurm.bats b/test/integration/slurm.bats index c52167f1..b6a88598 100644 --- a/test/integration/slurm.bats +++ b/test/integration/slurm.bats @@ -25,6 +25,16 @@ function setup() { # unexpected telemetry posts) export XDG_CONFIG_HOME=$(mktemp -d) + # force the system name to arapiles via the isolated user config. + # a system config (e.g. /etc/uenv/config.toml on Alps) sets system_name and + # takes priority over the CLUSTER_NAME environment variable, so the user + # config must set it to override the system one. tests that write their own + # config.toml append to this file (>>) to keep system_name. + mkdir -p "${XDG_CONFIG_HOME}/uenv" + cat > "${XDG_CONFIG_HOME}/uenv/config.toml" <> keeps the system_name written there) mkdir -p "${XDG_CONFIG_HOME}/uenv" printf '[elastic]\nurl = "http://127.0.0.1:%s"\n' "$elastic_port" \ - > "${XDG_CONFIG_HOME}/uenv/config.toml" + >> "${XDG_CONFIG_HOME}/uenv/config.toml" # run a full-setup srun (explicit --uenv triggers new-mount path which loads # config and populates config_g.elastic_config) @@ -351,7 +362,7 @@ EOF mkdir -p "${XDG_CONFIG_HOME}/uenv" printf '[elastic]\nurl = "http://127.0.0.1:%s"\n' "$elastic_port" \ - > "${XDG_CONFIG_HOME}/uenv/config.toml" + >> "${XDG_CONFIG_HOME}/uenv/config.toml" run uenv --repo="$RP" run --view=tool tool -- srun -n1 --oversubscribe echo hello assert_success @@ -384,7 +395,7 @@ EOF mkdir -p "${XDG_CONFIG_HOME}/uenv" printf '[elastic]\nurl = "http://127.0.0.1:%s"\n' "$elastic_port" \ - > "${XDG_CONFIG_HOME}/uenv/config.toml" + >> "${XDG_CONFIG_HOME}/uenv/config.toml" run_sbatch --repo="$RP" <<'EOF' #!/bin/bash @@ -419,7 +430,7 @@ EOF mkdir -p "${XDG_CONFIG_HOME}/uenv" printf '[elastic]\nurl = "http://127.0.0.1:%s"\n' "$elastic_port" \ - > "${XDG_CONFIG_HOME}/uenv/config.toml" + >> "${XDG_CONFIG_HOME}/uenv/config.toml" run_sbatch <base == "https://host.io"); REQUIRE(d->prefix == "a/b"); + // an explicit http scheme is preserved (e.g. a local test registry), and + // the port is carried through onto the base + auto e = split_registry("http://127.0.0.1:5000/uenv"); + REQUIRE(e.has_value()); + REQUIRE(e->base == "http://127.0.0.1:5000"); + REQUIRE(e->prefix == "uenv"); + // invalid: empty host / whitespace REQUIRE_FALSE(split_registry("").has_value()); REQUIRE_FALSE(split_registry("/uenv").has_value()); diff --git a/test/unit/oci_registry.cpp b/test/unit/oci_registry.cpp new file mode 100644 index 00000000..3e3faeed --- /dev/null +++ b/test/unit/oci_registry.cpp @@ -0,0 +1,245 @@ +// Live registry round-trip tests for the native OCI client. +// +// These drive push/pull/attach/referrers/copy against a real registry. A zot +// binary is started lazily on the first [registry] case and killed at process +// exit; each test uses a unique repository path, so OCI's per-repository +// namespacing keeps them isolated. If no zot binary is found the cases SKIP, so +// a normal `./test/unit` run is unaffected. +// +// The binary is located via -DUENV_TEST_ZOT_PATH (set by meson), the +// UENV_TEST_ZOT env override, or the PATH. + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +// locate the zot binary: build-time path, env override, then PATH. +std::string zot_binary() { + if (const char* e = std::getenv("UENV_TEST_ZOT")) { + if (::access(e, X_OK) == 0) { + return e; + } + } +#ifdef UENV_TEST_ZOT_PATH + if (::access(UENV_TEST_ZOT_PATH, X_OK) == 0) { + return UENV_TEST_ZOT_PATH; + } +#endif + // let execvp resolve "zot" on PATH; empty means "not found here". + return "zot"; +} + +int free_port() { + int fd = ::socket(AF_INET, SOCK_STREAM, 0); + sockaddr_in addr{}; + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + addr.sin_port = 0; + ::bind(fd, reinterpret_cast(&addr), sizeof(addr)); + socklen_t len = sizeof(addr); + ::getsockname(fd, reinterpret_cast(&addr), &len); + int port = ntohs(addr.sin_port); + ::close(fd); + return port; +} + +bool ready(const std::string& base) { + util::curl::request req; + req.url = base + "/v2/"; + auto resp = util::curl::perform(req); + return resp && (resp->status == 200 || resp->status == 401); +} + +// a lazily-started zot registry, shared by all [registry] cases and killed when +// the process exits. +struct zot_instance { + std::optional proc; + std::string base; // http://127.0.0.1:PORT, empty when unavailable + std::filesystem::path state; + + zot_instance() { + const auto zot = zot_binary(); + const int port = free_port(); + state = util::make_temp_dir(); + const auto cfg = state / "config.json"; + { + std::ofstream f{cfg}; + f << fmt::format( + R"({{"storage":{{"rootDirectory":"{}"}},)" + R"("http":{{"address":"127.0.0.1","port":"{}"}},)" + R"("log":{{"level":"error","output":"{}"}}}})", + (state / "registry").string(), port, + (state / "zot.log").string()); + } + auto p = util::run({zot, "serve", cfg.string()}); + if (!p) { + return; // no zot binary / failed to launch -> tests skip + } + const auto url = fmt::format("http://127.0.0.1:{}", port); + for (int i = 0; i < 150; ++i) { // up to ~30s + if (p->finished()) { + return; // zot died on startup + } + if (ready(url)) { + proc = std::move(*p); + base = url; + return; + } + ::usleep(200 * 1000); + } + p->kill(); + } + + ~zot_instance() { + if (proc) { + proc->kill(); + } + } +}; + +// the shared registry base URL, or empty when no registry could be started. +const std::string& registry_base() { + static zot_instance instance; + return instance.base; +} + +void write_file(const std::filesystem::path& path, std::string_view content) { + std::ofstream f{path, std::ios::binary}; + f << content; +} + +std::string read_file(const std::filesystem::path& path) { + std::ifstream f{path, std::ios::binary}; + return std::string{std::istreambuf_iterator(f), + std::istreambuf_iterator()}; +} + +} // namespace + +TEST_CASE("oci registry push/pull round-trip", "[registry]") { + const auto base = registry_base(); + if (base.empty()) { + SKIP("no zot binary available for the registry tests"); + } + + auto dir = util::make_temp_dir(); + const auto sqfs = dir / "store.squashfs"; + const std::string payload = "squashfs-bytes-round-trip"; + write_file(sqfs, payload); + + auto c = oci::client::create(base, "test/app/1.0"); + REQUIRE(c.has_value()); + + auto pushed = oci::push_squashfs(*c, sqfs, oci::reference::tag("v1")); + REQUIRE(pushed.has_value()); + + // the pushed digest is the sha256 of the manifest the registry stores. + auto resp = c->get_manifest(oci::reference::tag("v1")); + REQUIRE(resp.has_value()); + const auto local = oci::digest::from_sha256(util::sha256_string(resp->body)); + REQUIRE(local == *pushed); + if (resp->digest) { + REQUIRE(*resp->digest == *pushed); + } + + // pull the layer back; pull_squashfs self-verifies its digest internally. + auto image = oci::parse_manifest(resp->body); + REQUIRE(image.has_value()); + auto store = util::make_temp_dir(); + auto pulled = oci::pull_squashfs(*c, *image, store); + REQUIRE(pulled.has_value()); + REQUIRE(read_file(store / "store.squashfs") == payload); +} + +TEST_CASE("oci registry attach + referrers + pull_meta", "[registry]") { + const auto base = registry_base(); + if (base.empty()) { + SKIP("no zot binary available for the registry tests"); + } + + auto dir = util::make_temp_dir(); + const auto sqfs = dir / "store.squashfs"; + write_file(sqfs, "sqfs"); + const auto meta = dir / "meta"; + std::filesystem::create_directories(meta); + write_file(meta / "env.json", R"({"views":{}})"); + + auto c = oci::client::create(base, "test/meta-app/1.0"); + REQUIRE(c.has_value()); + + auto pushed = oci::push_squashfs(*c, sqfs, oci::reference::tag("v1")); + REQUIRE(pushed.has_value()); + + auto att = oci::attach(*c, oci::reference::digest(*pushed), "uenv/meta", meta); + REQUIRE(att.has_value()); + + auto refs = c->referrers(*pushed); + REQUIRE(refs.has_value()); + bool found_meta = false; + for (const auto& d : *refs) { + if (d.artifact_type && *d.artifact_type == "uenv/meta") { + found_meta = true; + } + } + REQUIRE(found_meta); + + auto store = util::make_temp_dir(); + auto got = oci::pull_meta(*c, *pushed, store); + REQUIRE(got.has_value()); + REQUIRE(*got); + REQUIRE(std::filesystem::exists(store / "meta" / "env.json")); + REQUIRE(read_file(store / "meta" / "env.json") == R"({"views":{}})"); +} + +TEST_CASE("oci registry copy preserves digest", "[registry]") { + const auto base = registry_base(); + if (base.empty()) { + SKIP("no zot binary available for the registry tests"); + } + + auto dir = util::make_temp_dir(); + const auto sqfs = dir / "store.squashfs"; + write_file(sqfs, "copy-me"); + + const std::string src_repo = "test/copy-src/1.0"; + const std::string dst_repo = "test/copy-dst/1.0"; + + auto src = oci::client::create(base, src_repo); + REQUIRE(src.has_value()); + auto pushed = oci::push_squashfs(*src, sqfs, oci::reference::tag("v1")); + REQUIRE(pushed.has_value()); + + auto copied = oci::copy_image(base, src_repo, dst_repo, *pushed, "v1", + std::nullopt); + REQUIRE(copied.has_value()); + + // the destination manifest is byte-identical, so its digest is preserved. + auto dst = oci::client::create(base, dst_repo); + REQUIRE(dst.has_value()); + auto resp = dst->get_manifest(oci::reference::tag("v1")); + REQUIRE(resp.has_value()); + const auto local = oci::digest::from_sha256(util::sha256_string(resp->body)); + REQUIRE(local == *pushed); +} diff --git a/test/unit/settings.cpp b/test/unit/settings.cpp index fc40f2ed..dcbd8a09 100644 --- a/test/unit/settings.cpp +++ b/test/unit/settings.cpp @@ -147,6 +147,33 @@ url = "https://my-elastic")"sv; REQUIRE(result->elastic_config); REQUIRE(result->elastic_config.value() == "https://my-elastic"); } + { + // registry config incl. the optional listing_url override + const std::string_view input = R"( +[registry] +url = "jfrog.svc.cscs.ch/uenv" +default_namespace = "deploy" +listing_url = "http://127.0.0.1:8080/list")"sv; + auto result = parse_config_toml(toml::parse(input), {}); + REQUIRE(result); + REQUIRE(result->registry); + REQUIRE(result->registry->url == "jfrog.svc.cscs.ch/uenv"); + REQUIRE(result->registry->default_namespace == "deploy"); + REQUIRE(result->registry->listing_url); + REQUIRE(result->registry->listing_url.value() == + "http://127.0.0.1:8080/list"); + } + { + // listing_url is optional + const std::string_view input = R"( +[registry] +url = "jfrog.svc.cscs.ch/uenv" +default_namespace = "deploy")"sv; + auto result = parse_config_toml(toml::parse(input), {}); + REQUIRE(result); + REQUIRE(result->registry); + REQUIRE(!result->registry->listing_url); + } { const std::string_view input = R"( [[repositories]] From 28045f11b14de660e75786018e5d05877b08ef76 Mon Sep 17 00:00:00 2001 From: bcumming Date: Sat, 4 Jul 2026 19:28:55 +0200 Subject: [PATCH 10/29] update testing info in CLAUDE.md --- CLAUDE.md | 49 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 4947353e..b0f3ceb4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -46,11 +46,12 @@ Configure via `-Doption=value` with `meson setup`: ## Testing -Four test suites exist: +Five test suites exist: 1. **unit** - C++ unit tests using Catch2 (in `test/unit/`) 2. **cli** - CLI integration tests using BATS (in `test/integration/cli.bats`) 3. **slurm** - Slurm plugin tests using BATS (in `test/integration/slurm.bats`) 4. **squashfs-mount** - setuid helper tests using BATS (in `test/integration/squashfs-mount.bats`) +5. **registry** - `uenv push`/`pull` against a throwaway local zot registry (in `test/integration/registry.bats`); self-skips when no zot binary is available ### Running Tests @@ -59,10 +60,30 @@ To run the tests, run the tests directly instead of running them through meson. ```bash # Run tests directly ./test/unit # Unit tests -./test/bats ./test/cli.bats # CLI tests -./test/bats ./test/slurm.bats # Slurm tests +./test/bats ./test/cli.bats # CLI tests +./test/bats ./test/slurm.bats # Slurm tests +./test/bats ./test/registry.bats # Registry (push/pull) tests ``` +### System name in tests + +Most CLI/Slurm tests resolve uenvs by a bare label (`app/42.0`, `tool`) and rely +on the default system being `arapiles` (the repo records are stored `@arapiles`). + +The default system name comes from the config layers, merged in this order (later +wins): `CLUSTER_NAME` env var → system config (`/etc/uenv/config.toml`) → user +config (`$XDG_CONFIG_HOME/uenv/config.toml` or `$HOME/.config/uenv/config.toml`) → +the `--system` CLI flag. On Alps the deployed system config sets +`system_name = 'eiger'`, which **overrides `CLUSTER_NAME=arapiles`**. So exporting +`CLUSTER_NAME` in a test is not sufficient. + +To force the system name, `cli.bats` and `slurm.bats` `setup()` write a throwaway +user config that sets `system_name = 'arapiles'` and point `XDG_CONFIG_HOME` at it +(user config beats system config). A test that writes its own `config.toml` must +include `system_name = 'arapiles'` (or append to the file created in `setup()` +with `>>` rather than clobbering it with `>`). Alternatively, pass `--system` on +the command line. + ### Elastic mock (`elastic_mock`) The slurm tests include an elastic telemetry test that uses a standalone mock server at `test/integration/elastic_mock`. The meson build copies it to `$BUILD_PATH/test/elastic_mock` (with execute permissions), and `setup_suite.bash` adds `$BUILD_PATH/test` to `PATH` so it is available by name in all BATS tests. @@ -88,6 +109,28 @@ elastic_mock kill /tmp/cap.json In BATS tests the helper functions in `common.bash` wrap the common lifecycle: `start_elastic_mock CAPTURE_FILE PORT` (backgrounds `serve` and calls `wait-server`), `stop_elastic_mock` (kills by PID), and `wait_elastic_post CAPTURE_FILE [TIMEOUT]`. +### Registry mock (`registry_ctl`, `listing_mock`) + +The `registry` suite exercises the native OCI client (`src/oci`) end-to-end against +a real registry, without containers or the old `oras` binary. Two helper scripts +are built into `$BUILD_PATH/test` (and so are on `PATH` in all BATS tests): + +- `registry_ctl` — manages the lifecycle of a throwaway [zot](https://zotregistry.dev) + registry (a single static binary). Subcommands include `runtime`/`zot` (report + the zot binary, empty if unavailable → suite self-skips), `free-port`, `serve + STATE PORT` (backgrounds itself), `wait PORT --timeout N`, and `kill STATE`. + The zot binary is fetched at build time by `test/integration/install-zot`. +- `listing_mock` — stands in for the CSCS uenv listing service + (`https://uenv-list.svc.cscs.ch/list`); same `free-port`/`serve`/`wait-server`/`kill` + lifecycle. Point uenv at it with `registry.listing_url` in the config. + +`common.bash` wraps these with `start_registry STATE PORT` / `stop_registry` and +`start_listing LISTING_FILE PORT` / `stop_listing`. `registry.bats` writes a user +`config.toml` whose `[registry]` block sets `url`, `default_namespace`, and +`listing_url` to point at the local zot + listing_mock. The `[registry]` unit tests +in `test/unit/oci_registry.cpp` cover the OCI round-trip directly by starting their +own zot. + ### Testing squashfs-mount The `squashfs-mount` helper requires setuid installation to test: From eee646f620ed9f0f5899e1c1d4683f946b86b89c Mon Sep 17 00:00:00 2001 From: bcumming Date: Sat, 4 Jul 2026 20:12:18 +0200 Subject: [PATCH 11/29] good bye oras --- CLAUDE.md | 7 +- meson.build | 25 --- meson_options.txt | 1 - packaging/uenv.spec | 1 - src/cli/uenv.cpp | 3 - src/oci/auth.h | 2 - src/uenv/oras.cpp | 362 -------------------------------------------- src/uenv/oras.h | 69 --------- src/util/fs.cpp | 21 --- src/util/fs.h | 4 - 10 files changed, 2 insertions(+), 493 deletions(-) delete mode 100644 src/uenv/oras.cpp delete mode 100644 src/uenv/oras.h diff --git a/CLAUDE.md b/CLAUDE.md index b0f3ceb4..265f477c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -42,7 +42,6 @@ Configure via `-Doption=value` with `meson setup`: - `cli=true|false` - Build CLI tool (default: true) - `slurm_plugin=true|false` - Build Slurm plugin (default: true) - `squashfs_mount=true|false` - Build squashfs-mount helper (default: false) -- `oras_version=X.Y.Z` - ORAS version to download (default: 1.2.0) ## Testing @@ -165,11 +164,11 @@ sudo meson install --destdir=$STAGE --no-rebuild --skip-subprojects - Parsing (`parse.h/cpp`, `lex.h` in util) - Mounting (`mount.h/cpp`) - Meta data (`meta.h/cpp`) - - Container registry interaction (`oras.h/cpp`) - Views (`view.h/cpp`) - Telemetry (`telemetry.h/cpp`, `elastic.h/cpp`) - Logging (`log.h/cpp`, `print.h/cpp`) - Settings management (`settings.h/cpp`) +- `src/oci/` - Native OCI registry client (container registry interaction: pull, push, copy, manifests, auth); replaces the external `oras` binary. See "Self-contained `src/oci`" below. - `src/util/` - Utility libraries (color, curl, envvars, fs, lex, lustre, semver, shell, signal, strings, subprocess, toml) - `src/site/` - Site-specific configuration (CSCS-specific logic) - `src/slurm/` - Slurm plugin implementation @@ -211,12 +210,10 @@ All dependencies are built as static libraries via meson wrap: - nlohmann_json - JSON parsing - sqlite3 - database - libcurl - HTTP operations +- zlib - gzip handling in the native OCI registry client (`src/oci`) - Catch2 - testing (when tests enabled) - barkeep - progress indicators (header-only in `extern/`) -The CLI build also downloads the ORAS binary for OCI registry operations. -ORAS is installed in `prefix/libexec` for runtime use by the CLI. - ## Development Notes ### Code Style diff --git a/meson.build b/meson.build index 37a70052..58f5fa28 100644 --- a/meson.build +++ b/meson.build @@ -9,8 +9,6 @@ project('uenv', ['cpp'], meson_version: '>=1.4') version = meson.project_version() -oras_version = get_option('oras_version') - uenv_slurm_plugin = get_option('slurm_plugin') uenv_cli = get_option('cli') uenv_squashfs_mount = get_option('squashfs_mount') @@ -70,7 +68,6 @@ lib_src = [ 'src/oci/pull.cpp', 'src/oci/push.cpp', 'src/oci/reference.cpp', - 'src/uenv/oras.cpp', 'src/uenv/parse.cpp', 'src/uenv/print.cpp', 'src/uenv/repository.cpp', @@ -148,28 +145,6 @@ if uenv_cli ], install: true) - arch = 'amd64' - if host_machine.cpu_family() == 'aarch64' - arch = 'arm64' - endif - oras_url = 'https://github.com/oras-project/oras/releases/download/v@0@/oras_@0@_linux_@1@.tar.gz'.format(oras_version, arch) - - oras_download = custom_target( - 'oras-download', - output: 'oras.tar.gz', - command: ['wget', '--quiet', oras_url, '-O', '@OUTPUT@'], - install: false, - ) - - oras_extract = custom_target( - 'oras-extract', - input: oras_download, - output: 'oras', - command: ['tar', 'xf', '@INPUT@', 'oras'], - install: true, - install_dir: get_option('libexecdir'), - ) - meson.add_install_script('./meson-scripts/install-bash-completion.sh') endif diff --git a/meson_options.txt b/meson_options.txt index c97af130..66e322cb 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -4,6 +4,5 @@ option('slurm_plugin', type: 'boolean', value: true) option('cli', type: 'boolean', value: true) option('squashfs_mount', type: 'boolean', value: false) -option('oras_version', type: 'string', value: '1.2.0') #option('slurm_version', type: 'string', value: '24.05.0') option('slurm_version', type: 'string', value: '0.00.0') diff --git a/packaging/uenv.spec b/packaging/uenv.spec index 4a743384..c2005810 100644 --- a/packaging/uenv.spec +++ b/packaging/uenv.spec @@ -35,5 +35,4 @@ echo "$REQ" > "$CNF" %{_bindir}/uenv %{_bindir}/squashfs-mount %attr(4755, root, root) %{_bindir}/squashfs-mount -/usr/libexec/oras /usr/share/bash-completion/completions/uenv diff --git a/src/cli/uenv.cpp b/src/cli/uenv.cpp index e6390351..53f0e740 100644 --- a/src/cli/uenv.cpp +++ b/src/cli/uenv.cpp @@ -99,9 +99,6 @@ int main(int argc, char** argv) { if (auto bin = util::exe_path()) { spdlog::info("using uenv {}", bin->string()); } - if (auto oras = util::oras_path()) { - spdlog::info("using oras {}", oras->string()); - } // print the version and exit if the --version flag was passed if (print_version) { diff --git a/src/oci/auth.h b/src/oci/auth.h index 8abb7385..201db27d 100644 --- a/src/oci/auth.h +++ b/src/oci/auth.h @@ -10,8 +10,6 @@ namespace oci { // HTTP basic-auth credentials used to obtain a push/private-pull token. -// (defined here rather than reusing oras::credentials so the oci client does -// not depend on the oras module it is meant to replace.) struct credentials { std::string username; std::string password; // password or personal access token diff --git a/src/uenv/oras.cpp b/src/uenv/oras.cpp deleted file mode 100644 index 41ad0879..00000000 --- a/src/uenv/oras.cpp +++ /dev/null @@ -1,362 +0,0 @@ -#include - -#include -#include -#include - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -namespace uenv { -namespace oras { - -using opt_creds = std::optional; - -// stores the "result" of calling the oras cli tool as an external process. -struct oras_output { - // return code of the oras call - int returncode = -1; - - // the full captured stdout output - std::string stdout; - - // the full captured stderr output - std::string stderr; -}; - -constexpr auto generic_error_message = - "unknown error - rerun with the -vv flag and send an error report to the " - "CSCS service desk."; - -error generic_error(std::string err) { - return {-1, std::move(err), generic_error_message}; -} - -// Convert oras_output to a uenv::oras::error. -// oras returns 1 for all errors that it bothers signalling (from what we -// observe - there is no documentation). sometimes it returns 0, and you have to -// parse stderr to double check. -// Hence, parse stderr & stdout for hints about -// what went wrong to generate the user-friendly message -error create_error(const oras_output& result) { - std::string message = generic_error_message; - - auto contains = [](std::string_view string, - std::string_view contents) -> bool { - return string.find(contents) != std::string_view::npos; - }; - std::string_view err = result.stderr; - if (contains(err, "403") && contains(err, "Forbidden")) { - return { - 403, result.stderr, - "Invalid credentials were provided, or you may not have permission " - "to perform the requested action.\nTry using the --token flag if " - "you are trying to access restricted software.\nCSCS staff can " - "configure oras to use their credentials."}; - } - if (contains(err, "Token failed verification: parse") && - contains(err, "Error response from registry")) { - return { - 403, result.stderr, - "The token failed parsing. It may be invalid, the wrong token,\n" - "or need to be regenerated."}; - } - if (contains(err, "Wrong username was used") && - contains(err, "Error response from registry")) { - return {403, result.stderr, - "Invalid username was provided. Check the --username flag."}; - } - if (contains(err, "Token failed verification: revoked") && - contains(err, "Error response from registry")) { - return {403, result.stderr, - "The registry token has been revoked. Create a new token, and " - "either pass it as an input file, or update the entry in " - "~/.docker/config.json."}; - } - if (contains(err, "unauthorized") && - contains(err, "Error response from registry")) { - return {403, result.stderr, - fmt::format("no authorization: provide valid --token and " - "--username arguments.", - err)}; - } - if (result.returncode == 0) { - return {0, {}, {}}; - } - return {result.returncode, result.stderr, std::move(message)}; -}; - -std::vector -redact_arguments(const std::vector& args) { - std::vector redacted_args{}; - bool redact_next = false; - for (auto arg : args) { - if (redact_next) { - redacted_args.push_back(std::string(arg.size(), 'X')); - redact_next = false; - } else if (arg.find("password") != std::string::npos) { - if (auto eqpos = arg.find("="); eqpos != std::string::npos) { - redacted_args.push_back( - std::string(arg.begin(), arg.begin() + eqpos + 1) + - std::string(arg.size() - eqpos, 'X')); - } else { - redacted_args.push_back(arg); - redact_next = true; - } - } else { - redacted_args.push_back(arg); - } - } - - return redacted_args; -} - -oras_output -run_oras(std::vector args, - std::optional runpath = std::nullopt) { - auto oras = util::oras_path(); - if (!oras) { - spdlog::error("no oras executable found"); - return {.returncode = -1}; - } - - args.insert(args.begin(), oras->string()); - spdlog::trace("run_oras: {}", fmt::join(redact_arguments(args), " ")); - auto proc = util::run(args, runpath); - - auto result = proc->wait(); - - return {.returncode = result, - .stdout = proc->out.string(), - .stderr = proc->err.string()}; -} - -util::expected -run_oras_async(std::vector args, - std::optional runpath = std::nullopt) { - auto oras = util::oras_path(); - if (!oras) { - return util::unexpected{"no oras executable found"}; - } - - args.insert(args.begin(), oras->string()); - spdlog::trace("run_oras_async: {}", fmt::join(redact_arguments(args), " ")); - - return util::run(args, runpath); -} - -util::expected push_tag(const std::string& registry, - const std::string& nspace, - const uenv_label& label, - const std::filesystem::path& source, - const std::optional token) { - using namespace std::chrono_literals; - namespace fs = std::filesystem; - namespace bk = barkeep; - - // Format the registry address with the appropriate path structure - auto address = - fmt::format("{}/{}/{}/{}/{}/{}:{}", registry, nspace, *label.system, - *label.uarch, *label.name, *label.version, *label.tag); - - spdlog::debug("oras::push_tag: {}", address); - - // Prepare the arguments for the push command - std::vector args{"push", "--concurrency", "10"}; - - if (token) { - args.push_back("--password"); - args.push_back(token->token); - args.push_back("--username"); - args.push_back(token->username); - } - - // Add artifact type and annotations - args.push_back("--artifact-type"); - args.push_back("application/x-squashfs"); - - // Add the destination address - args.push_back(address); - // Add the file to push - note that we strip the absolute and relative path - args.push_back("./" + source.filename().string()); - - // Run the oras command asynchronously - // Note: oras must be called in the path of the file - auto proc = run_oras_async(args, source.parent_path()); - - if (!proc) { - spdlog::error("unable to push tag with oras: {}", proc.error()); - return util::unexpected{generic_error(proc.error())}; - } - - // Create a spinner to show upload progress - auto spinner = bk::Animation({ - .message = fmt::format("pushing {} to registry", - fs::path(source).filename().string()), - .style = bk::Ellipsis, - .no_tty = !isatty(fileno(stdout)), - }); - - // Handle signals during upload (e.g., Ctrl+C) - util::set_signal_catcher(); - - // Loop until the upload process completes - while (!proc->finished()) { - std::this_thread::sleep_for(100ms); - - // Handle interruption signals - if (util::signal_raised()) { - spdlog::error("signal raised - interrupting upload"); - proc->kill(); - throw util::signal_exception(util::last_signal_raised()); - } - } - - spinner->done(); - - // Check if the upload was successful - auto err = create_error({.returncode = proc->rvalue(), - .stdout = proc->out.string(), - .stderr = proc->err.string()}); - if (err.returncode) { - spdlog::error("unable to push tag with oras: {}", proc->err.string()); - return util::unexpected{err}; - } - - return {}; -} - -util::expected push_meta(const std::string& registry, - const std::string& nspace, - const uenv_label& label, - const std::filesystem::path& meta_path, - const std::optional token) { - using namespace std::chrono_literals; - namespace fs = std::filesystem; - namespace bk = barkeep; - - // Check if the meta path exists and is a directory - if (!fs::exists(meta_path) || !fs::is_directory(meta_path)) { - spdlog::error( - "metadata directory {} does not exist or is not a directory", - meta_path.string()); - return util::unexpected{error( - 1, "metadata directory not found", - fmt::format( - "metadata directory {} does not exist or is not a directory", - meta_path.string()))}; - } - - // Format the registry address with the appropriate path structure - auto address = - fmt::format("{}/{}/{}/{}/{}/{}:{}", registry, nspace, *label.system, - *label.uarch, *label.name, *label.version, *label.tag); - - spdlog::debug("oras::push_meta: {}", address); - - // Prepare the arguments for the attach command - std::vector args{"attach"}; - - if (token) { - args.push_back("--password"); - args.push_back(token->token); - args.push_back("--username"); - args.push_back(token->username); - } - - // Add artifact type and annotations for metadata - args.push_back("--artifact-type"); - args.push_back("uenv/meta"); - - // Add destination address - args.push_back(address); - // Add the meta data path - strip relative/absolute path from the path - args.push_back("./" + meta_path.filename().string()); - - // Run the oras command asynchronously - // Note: oras must be called in the path of the file - auto proc = run_oras_async(args, meta_path.parent_path()); - - if (!proc) { - spdlog::error("unable to push metadata with oras: {}", proc.error()); - return util::unexpected{generic_error(proc.error())}; - } - - // Create a spinner to show upload progress - auto spinner = bk::Animation({ - .message = fmt::format("pushing metadata to registry"), - .style = bk::Ellipsis, - .no_tty = !isatty(fileno(stdout)), - }); - - // Handle signals during upload (e.g., Ctrl+C) - util::set_signal_catcher(); - - // Loop until the upload process completes - while (!proc->finished()) { - std::this_thread::sleep_for(100ms); - - // Handle interruption signals - if (util::signal_raised()) { - spdlog::error("signal raised - interrupting metadata upload"); - proc->kill(); - throw util::signal_exception(util::last_signal_raised()); - } - } - - spinner->done(); - - // Check if the upload was successful - auto err = create_error({.returncode = proc->rvalue(), - .stdout = proc->out.string(), - .stderr = proc->err.string()}); - if (err.returncode) { - spdlog::error("unable to metadata with oras: {}", proc->err.string()); - return util::unexpected{err}; - } - - return {}; -} - -util::expected -copy(const std::string& registry, const std::string& src_nspace, - const uenv_record& src_uenv, const std::string& dst_nspace, - const uenv_record& dst_uenv, const std::optional token) { - - auto address = [®istry](auto& nspace, auto& record) -> std::string { - return fmt::format("{}/{}/{}/{}/{}/{}:{}", registry, nspace, - record.system, record.uarch, record.name, - record.version, record.tag); - }; - const auto src_url = address(src_nspace, src_uenv); - const auto dst_url = address(dst_nspace, dst_uenv); - - std::vector args = {"cp", "--concurrency", "10", - "--recursive", src_url, dst_url}; - if (token) { - args.push_back(fmt::format("--from-password={}", token->token)); - args.push_back(fmt::format("--from-username={}", token->username)); - args.push_back(fmt::format("--to-password={}", token->token)); - args.push_back(fmt::format("--to-username={}", token->username)); - } - auto result = run_oras(args); - - if (result.returncode) { - spdlog::error("oras cp {}: {}", result.returncode, result.stderr); - return util::unexpected{create_error(result)}; - } - - return {}; -} - -} // namespace oras -} // namespace uenv diff --git a/src/uenv/oras.h b/src/uenv/oras.h deleted file mode 100644 index 96180f38..00000000 --- a/src/uenv/oras.h +++ /dev/null @@ -1,69 +0,0 @@ -#pragma once - -#include -#include -#include - -#include -#include - -namespace uenv { -namespace oras { - -struct credentials { - std::string username; - std::string token; -}; - -struct error { - error(std::string_view msg) : message(msg) { - } - error(int code, std::string_view err, std::string_view msg) - : returncode(code), stderr(err), message(msg) { - } - error() = default; - - int returncode = 0; - std::string stderr = {}; - std::string message = {}; - - operator bool() const { - return returncode == 0; - }; -}; - -util::expected -push_tag(const std::string& registry, const std::string& nspace, - const uenv_label& label, const std::filesystem::path& source, - const std::optional token = std::nullopt); - -util::expected -push_meta(const std::string& registry, const std::string& nspace, - const uenv_label& label, const std::filesystem::path& meta_path, - const std::optional token = std::nullopt); - -util::expected -copy(const std::string& registry, const std::string& src_nspace, - const uenv_record& src_uenv, const std::string& dst_nspace, - const uenv_record& dst_uenv, - const std::optional token = std::nullopt); - -} // namespace oras -} // namespace uenv - -#include -template <> class fmt::formatter { - public: - // parse format specification and store it: - constexpr auto parse(format_parse_context& ctx) { - return ctx.end(); - } - // format a value using stored specification: - template - constexpr auto format(uenv::oras::credentials const& c, - FmtContext& ctx) const { - // replace token characters with 'X' - return fmt::format_to(ctx.out(), "{{username: {}, token: {:X>{}}}}", - c.username, "", c.token.size()); - } -}; diff --git a/src/util/fs.cpp b/src/util/fs.cpp index 55b93550..4cb37c4e 100644 --- a/src/util/fs.cpp +++ b/src/util/fs.cpp @@ -187,27 +187,6 @@ std::optional exe_path() { return p; } -std::optional oras_path() { - namespace fs = std::filesystem; - - auto exe = exe_path(); - if (!exe) { - return std::nullopt; - } - - const auto prefix = exe->parent_path(); - - for (auto& path : {"../libexec/oras", "oras"}) { - const auto p = prefix / path; - if (fs::is_regular_file(p)) { - return fs::canonical(p); - } - } - - // maybe this could be extended to search PATH - return std::nullopt; -} - file_level file_access_level(const std::filesystem::path& path) { namespace fs = std::filesystem; diff --git a/src/util/fs.h b/src/util/fs.h index ae64a101..59ead985 100644 --- a/src/util/fs.h +++ b/src/util/fs.h @@ -49,10 +49,6 @@ make_file_lock(const std::filesystem::path& path); // returns empty if there is an error std::optional exe_path(); -// return the path of the oras executable -// returns empty if it can't be found -std::optional oras_path(); - // for determining the level of access to a file or directory // if there is an error, or the file does not exist `none` is // returned. From efe666b9b79aa8bb3b0044ad0b7efeba71b06d86 Mon Sep 17 00:00:00 2001 From: bcumming Date: Sun, 5 Jul 2026 08:19:00 +0200 Subject: [PATCH 12/29] hash downloaded blobs while streaming --- src/oci/client.cpp | 30 ++++++++++++++++++++++++++++++ src/oci/pull.cpp | 18 ++---------------- src/util/curl.cpp | 20 +++++++++++++++++--- src/util/curl.h | 6 ++++++ 4 files changed, 55 insertions(+), 19 deletions(-) diff --git a/src/oci/client.cpp b/src/oci/client.cpp index 6c0f74ae..f67ddce9 100644 --- a/src/oci/client.cpp +++ b/src/oci/client.cpp @@ -1,5 +1,8 @@ +#include +#include #include #include +#include #include #include #include @@ -10,6 +13,7 @@ #include #include +#include #include #include @@ -274,6 +278,19 @@ client::get_blob_to_file( req.on_download_progress = std::move(progress); req.should_abort = std::move(should_abort); + // hash the blob as it streams to disk so its content can be verified against + // the requested digest without a second read pass over a multi-GB file. + const bool verify = d.algorithm() == "sha256"; + util::sha256_state hash; + if (verify) { + util::sha256_init(hash); + req.on_download_data = [&hash](const char* p, std::size_t n) { + util::sha256_update( + hash, std::span{ + reinterpret_cast(p), n}); + }; + } + auto resp = util::curl::perform(req); if (!resp) { return util::unexpected{resp.error().message}; @@ -282,6 +299,19 @@ client::get_blob_to_file( return util::unexpected{fmt::format( "failed to fetch blob {} (status {})", d.string(), resp->status)}; } + + if (verify) { + const auto got = digest::from_sha256(util::sha256_final(hash)); + if (got != d) { + std::error_code ec; + std::filesystem::remove(file, ec); + return util::unexpected{ + fmt::format("downloaded blob digest mismatch: expected {}, " + "got {}", + d.string(), got.string())}; + } + spdlog::debug("oci::get_blob_to_file verified {}", got.string()); + } return {}; } diff --git a/src/oci/pull.cpp b/src/oci/pull.cpp index 24df5b2f..479ada1d 100644 --- a/src/oci/pull.cpp +++ b/src/oci/pull.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include namespace oci { @@ -90,26 +89,13 @@ pull_squashfs(client& c, const manifest& image, const fs::path& store, const fs::path dest = store / "store.squashfs"; spdlog::debug("oci::pull_squashfs {} -> {}", want.string(), dest.string()); + // get_blob_to_file streams the layer to disk and verifies its digest on the + // fly (hashing as it downloads). if (auto ok = c.get_blob_to_file(want, dest, std::move(progress), std::move(should_abort)); !ok) { return util::unexpected{ok.error()}; } - - // self-verify: the on-disk file must re-digest to the layer digest. - auto actual = util::sha256_file(dest); - if (!actual) { - return util::unexpected{actual.error()}; - } - auto got = digest::from_sha256(*actual); - if (got != want) { - std::error_code ec; - fs::remove(dest, ec); - return util::unexpected{fmt::format( - "downloaded squashfs digest mismatch: expected {}, got {}", - want.string(), got.string())}; - } - spdlog::debug("oci::pull_squashfs verified {}", got.string()); return {}; } diff --git a/src/util/curl.cpp b/src/util/curl.cpp index 8b8164f3..06072461 100644 --- a/src/util/curl.cpp +++ b/src/util/curl.cpp @@ -247,10 +247,21 @@ size_t header_callback(char* buffer, size_t size, size_t nitems, return total; } +// download sink for the file path: the FILE* to write to, plus an optional tap +// (request::on_download_data) that observes each chunk written. +struct file_sink { + FILE* file; + const std::function* on_data; +}; + size_t file_write_callback(void* source, size_t size, size_t n, void* target) { const size_t realsize = size * n; - FILE* f = static_cast(target); - return fwrite(source, 1, realsize, f); + auto& sink = *static_cast(target); + const size_t written = fwrite(source, 1, realsize, sink.file); + if (written > 0 && sink.on_data && *sink.on_data) { + (*sink.on_data)(static_cast(source), written); + } + return written; } int xferinfo_callback(void* p, curl_off_t dltotal, curl_off_t dlnow, @@ -382,6 +393,7 @@ expected perform(const request& req) { // downloads) or buffered in memory. std::vector body; FILE* download = nullptr; + file_sink download_sink{}; auto cleanup_download = defer([&download]() { if (download) { fclose(download); @@ -395,9 +407,11 @@ expected perform(const request& req) { fmt::format("unable to open {} for download", req.download_file->string())}}; } + download_sink.file = download; + download_sink.on_data = &req.on_download_data; CURL_EASY( curl_easy_setopt(h, CURLOPT_WRITEFUNCTION, file_write_callback)); - CURL_EASY(curl_easy_setopt(h, CURLOPT_WRITEDATA, (void*)download)); + CURL_EASY(curl_easy_setopt(h, CURLOPT_WRITEDATA, (void*)&download_sink)); } else { body.reserve(200000); CURL_EASY(curl_easy_setopt(h, CURLOPT_WRITEFUNCTION, memory_callback)); diff --git a/src/util/curl.h b/src/util/curl.h index 55dfb76f..633e8cac 100644 --- a/src/util/curl.h +++ b/src/util/curl.h @@ -68,6 +68,12 @@ struct request { // bytes_total is 0 until the server reports a content length. std::function on_download_progress; + // optional tap on the response body as it streams to download_file: called + // with each chunk actually written, before returning to libcurl. lets a + // caller hash the blob on the fly and so verify its digest without a second + // read pass over a multi-GB file. only invoked on the download_file path. + std::function on_download_data; + // optional abort predicate, polled during transfer; returning true aborts // the request (used to make a long download responsive to Ctrl-C). std::function should_abort; From 173f52b9903bec7e5857ac85809c72ebc3b0deaa Mon Sep 17 00:00:00 2001 From: bcumming Date: Sun, 5 Jul 2026 08:46:38 +0200 Subject: [PATCH 13/29] harden fs operations --- src/cli/build.cpp | 7 +++++- src/oci/auth.cpp | 7 +++--- src/oci/client.cpp | 15 ++++++++++++ src/oci/pull.cpp | 31 ++++++++++------------- src/oci/push.cpp | 26 +++++++++++++++----- src/util/fs.cpp | 45 ++++++++++++++++++++++++++++++---- src/util/fs.h | 8 +++++- test/unit/fs.cpp | 50 +++++++++++++++++++++++++++++++++++--- test/unit/mount.cpp | 6 ++--- test/unit/oci_auth.cpp | 8 +++--- test/unit/oci_registry.cpp | 12 ++++----- test/unit/repository.cpp | 4 +-- 12 files changed, 167 insertions(+), 52 deletions(-) diff --git a/src/cli/build.cpp b/src/cli/build.cpp index 4b694a42..1a8a4a19 100644 --- a/src/cli/build.cpp +++ b/src/cli/build.cpp @@ -100,7 +100,12 @@ int build(const build_args& args, return 1; } - auto recipe_tar_path = util::make_temp_dir() / "recipe.tar.gz"; + auto tmp_dir = util::make_temp_dir(); + if (!tmp_dir) { + term::error("{}", tmp_dir.error()); + return 1; + } + auto recipe_tar_path = *tmp_dir / "recipe.tar.gz"; auto proc = util::run({"env", "--chdir", recipe_path.string(), "tar", "--dereference", "-czf", recipe_tar_path, "."}); if (proc->rvalue() > 0) { diff --git a/src/oci/auth.cpp b/src/oci/auth.cpp index e9972b38..52a00c77 100644 --- a/src/oci/auth.cpp +++ b/src/oci/auth.cpp @@ -162,14 +162,15 @@ get_credentials(std::optional username, } fs::path token_path{token.value()}; - if (!fs::exists(token_path)) { + std::error_code ec; + if (!fs::exists(token_path, ec)) { return util::unexpected{fmt::format( "the token '{}' is not a path or file.", token_path.string())}; } - if (fs::is_directory(token_path)) { + if (fs::is_directory(token_path, ec)) { token_path = token_path / "TOKEN"; - if (!fs::exists(token_path)) { + if (!fs::exists(token_path, ec)) { return util::unexpected{fmt::format( "the token file '{}' does not exist.", token_path.string())}; } diff --git a/src/oci/client.cpp b/src/oci/client.cpp index f67ddce9..b4d52566 100644 --- a/src/oci/client.cpp +++ b/src/oci/client.cpp @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -266,6 +267,16 @@ client::get_blob_to_file( const digest& d, const std::filesystem::path& file, std::function progress, std::function should_abort) { + // check the destination directory is writable up front, so a bad target + // surfaces as a clear error rather than a curl write failure. + const auto parent = file.parent_path(); + if (!parent.empty() && + util::file_access_level(parent) != util::file_level::readwrite) { + return util::unexpected{fmt::format( + "cannot write blob to {}: {} is not a writable directory", + file.string(), parent.string())}; + } + auto token = token_for(false); if (!token) { return util::unexpected{token.error()}; @@ -482,6 +493,10 @@ client::mount_blob(const digest& d, const std::string& from_repository) { util::expected client::put_blob(const digest& d, const std::filesystem::path& file) { + if (util::file_access_level(file) < util::file_level::readonly) { + return util::unexpected{fmt::format( + "cannot upload blob: {} is not a readable file", file.string())}; + } if (auto exists = blob_exists(d); exists && *exists) { spdlog::trace("oci::put_blob {} already present", d.string()); return {}; diff --git a/src/oci/pull.cpp b/src/oci/pull.cpp index 479ada1d..54205aa4 100644 --- a/src/oci/pull.cpp +++ b/src/oci/pull.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include namespace oci { @@ -24,7 +25,13 @@ constexpr std::string_view squashfs_title = "store.squashfs"; // extract a gzipped tar (held in memory) into `dest`, using the system tar. util::expected extract_targz(const std::string& data, const fs::path& dest) { - auto tmp = dest / ".uenv-meta.tar.gz"; + // stage the archive in a private temp dir (unique name, not a predictable + // path inside `dest`) before handing it to tar. + auto work = util::make_temp_dir(); + if (!work) { + return util::unexpected{work.error()}; + } + auto tmp = *work / "meta.tar.gz"; { std::ofstream out(tmp, std::ios::binary | std::ios::trunc); if (!out) { @@ -38,14 +45,15 @@ util::expected extract_targz(const std::string& data, } } + std::error_code ec; auto proc = util::run({"tar", "-xzf", tmp.string(), "-C", dest.string()}); if (!proc) { - fs::remove(tmp); + fs::remove(tmp, ec); return util::unexpected{ fmt::format("unable to run tar to unpack meta: {}", proc.error())}; } auto rc = proc->wait(); - fs::remove(tmp); + fs::remove(tmp, ec); if (rc != 0) { return util::unexpected{ fmt::format("tar failed to unpack meta (exit {})", rc)}; @@ -53,19 +61,6 @@ util::expected extract_targz(const std::string& data, return {}; } -util::expected ensure_dir(const fs::path& store) { - if (fs::exists(store)) { - return {}; - } - std::error_code ec; - fs::create_directories(store, ec); - if (ec) { - return util::unexpected{ - fmt::format("unable to create {}: {}", store.string(), ec.message())}; - } - return {}; -} - } // namespace util::expected @@ -83,7 +78,7 @@ pull_squashfs(client& c, const manifest& image, const fs::path& store, } const digest& want = layer->digest; - if (auto ok = ensure_dir(store); !ok) { + if (auto ok = util::ensure_directory(store); !ok) { return util::unexpected{ok.error()}; } @@ -142,7 +137,7 @@ pull_meta(client& c, const digest& manifest_digest, const fs::path& store) { return util::unexpected{blob.error()}; } - if (auto ok = ensure_dir(store); !ok) { + if (auto ok = util::ensure_directory(store); !ok) { return util::unexpected{ok.error()}; } if (auto ok = extract_targz(*blob, store); !ok) { diff --git a/src/oci/push.cpp b/src/oci/push.cpp index 6ff42cda..1dd28877 100644 --- a/src/oci/push.cpp +++ b/src/oci/push.cpp @@ -79,7 +79,10 @@ package_directory(const fs::path& dir) { const auto name = dir.filename().string(); auto work = util::make_temp_dir(); - const auto tar_path = work / "payload.tar"; + if (!work) { + return util::unexpected{work.error()}; + } + const auto tar_path = *work / "payload.tar"; auto tar = util::run({"tar", "--sort=name", "--format=posix", "--mtime=@0", "--owner=0", "--group=0", "--numeric-owner", "-cf", @@ -130,7 +133,7 @@ package_directory(const fs::path& dir) { tar_digest->string()}, {std::string{annotation_unpack}, "true"}, {std::string{annotation_title}, name}}}, - .scratch = work}; + .scratch = *work}; } // Package a single file as a raw blob layer (no unpack), titled with its @@ -190,7 +193,10 @@ copy_blob(client& src, client& dst, const std::string& from_repo, // fallback: pull the blob to a scratch file, then push it. spdlog::debug("copy_blob streaming {} (mount declined)", d.string()); auto work = util::make_temp_dir(); - const auto tmp = work / "blob"; + if (!work) { + return util::unexpected{work.error()}; + } + const auto tmp = *work / "blob"; if (auto ok = src.get_blob_to_file(d, tmp); !ok) { return util::unexpected{ok.error()}; } @@ -279,10 +285,19 @@ push_squashfs(client& c, const fs::path& squashfs, const reference& ref) { util::expected attach(client& c, const reference& subject, std::string_view artifact_type, const fs::path& payload) { - if (!fs::exists(payload)) { + // stat the payload once, and branch file-vs-directory off that single + // result (avoids a TOCTOU between exists() and is_directory()). + std::error_code ec; + auto payload_status = fs::status(payload, ec); + if (ec || !fs::exists(payload_status)) { return util::unexpected{ fmt::format("payload {} does not exist", payload.string())}; } + const bool payload_is_dir = fs::is_directory(payload_status); + if (util::file_access_level(payload) < util::file_level::readonly) { + return util::unexpected{ + fmt::format("payload {} is not readable", payload.string())}; + } // resolve the subject (the image we annotate). auto subj = c.get_manifest(subject); @@ -298,8 +313,7 @@ attach(client& c, const reference& subject, std::string_view artifact_type, // package the payload. util::expected packaged = - fs::is_directory(payload) ? package_directory(payload) - : package_file(payload); + payload_is_dir ? package_directory(payload) : package_file(payload); if (!packaged) { return util::unexpected{packaged.error()}; } diff --git a/src/util/fs.cpp b/src/util/fs.cpp index 4cb37c4e..cff70ddb 100644 --- a/src/util/fs.cpp +++ b/src/util/fs.cpp @@ -1,3 +1,5 @@ +#include +#include #include #include #include @@ -55,14 +57,19 @@ bool is_temp_dir(const std::filesystem::path& path) { return false; } -std::filesystem::path make_temp_dir() { +util::expected make_temp_dir() { namespace fs = std::filesystem; auto tmp_template = fs::temp_directory_path().string() + "/uenv-XXXXXXXXXXXX"; std::vector base(tmp_template.data(), tmp_template.data() + tmp_template.size() + 1); - fs::path tmp_path = mkdtemp(base.data()); + if (mkdtemp(base.data()) == nullptr) { + return unexpected(fmt::format("unable to create a temporary directory " + "from template {}: {}", + tmp_template, strerror(errno))); + } + fs::path tmp_path = base.data(); spdlog::debug("make_temp_dir: created {}", tmp_path.string(), fs::is_directory(tmp_path)); @@ -72,6 +79,30 @@ std::filesystem::path make_temp_dir() { return tmp_path; } +util::expected +ensure_directory(const std::filesystem::path& path) { + namespace fs = std::filesystem; + + std::error_code ec; + fs::create_directories(path, ec); + if (ec) { + return unexpected( + fmt::format("unable to create {}: {}", path.string(), ec.message())); + } + + if (!fs::is_directory(path, ec)) { + return unexpected(fmt::format( + "{} exists but is not a directory", path.string())); + } + + if (file_access_level(path) != file_level::readwrite) { + return unexpected( + fmt::format("the directory {} is not writable", path.string())); + } + + return {}; +} + util::expected unsquashfs_tmp(const std::filesystem::path& sqfs, const std::filesystem::path& contents) { @@ -83,8 +114,12 @@ unsquashfs_tmp(const std::filesystem::path& sqfs, } auto base = make_temp_dir(); + if (!base) { + return unexpected( + fmt::format("unsquashfs_tmp: {}", base.error())); + } std::vector command{ - "unsquashfs", "-no-exit", "-d", base.string(), + "unsquashfs", "-no-exit", "-d", base->string(), // single threaded to avoid resource contention. "-processors", "1", sqfs.string(), contents.string()}; spdlog::debug("unsquashfs_tmp: attempting to unpack {} from {}", @@ -111,8 +146,8 @@ unsquashfs_tmp(const std::filesystem::path& sqfs, } spdlog::info("unsquashfs_tmp: unpacked {} from {} to {}", contents.string(), - sqfs.string(), base.string()); - return base; + sqfs.string(), base->string()); + return *base; } util::expected diff --git a/src/util/fs.h b/src/util/fs.h index 59ead985..b48fc1da 100644 --- a/src/util/fs.h +++ b/src/util/fs.h @@ -8,10 +8,16 @@ namespace util { -std::filesystem::path make_temp_dir(); +util::expected make_temp_dir(); bool is_temp_dir(const std::filesystem::path& path); +// Ensure `path` is a writable directory, creating it (and any missing parents) +// if necessary. Succeeds only when the final path exists, is a directory, and +// is writable. +util::expected +ensure_directory(const std::filesystem::path& path); + util::expected unsquashfs_tmp(const std::filesystem::path& sqfs, const std::filesystem::path& contents); diff --git a/test/unit/fs.cpp b/test/unit/fs.cpp index 51970f62..35c9df03 100644 --- a/test/unit/fs.cpp +++ b/test/unit/fs.cpp @@ -11,9 +11,13 @@ namespace fs = std::filesystem; TEST_CASE("make_temp_dir", "[fs]") { - auto dir1 = util::make_temp_dir(); + auto r1 = util::make_temp_dir(); + REQUIRE(r1); + auto dir1 = *r1; REQUIRE(fs::is_directory(dir1)); - auto dir2 = util::make_temp_dir(); + auto r2 = util::make_temp_dir(); + REQUIRE(r2); + auto dir2 = *r2; REQUIRE(dir1 != dir2); REQUIRE(util::is_temp_dir(dir1)); @@ -24,6 +28,46 @@ TEST_CASE("make_temp_dir", "[fs]") { REQUIRE(!util::is_temp_dir("/scratch/bar")); } +TEST_CASE("ensure_directory", "[fs]") { + auto base = util::make_temp_dir(); + REQUIRE(base); + + // creates a new (nested) directory + { + auto d = *base / "a" / "b" / "c"; + REQUIRE(util::ensure_directory(d)); + REQUIRE(fs::is_directory(d)); + } + + // succeeds when the directory already exists + { + auto d = *base / "exists"; + REQUIRE(util::ensure_directory(d)); + REQUIRE(util::ensure_directory(d)); + } + + // fails when the path exists but is a regular file + { + auto p = *base / "afile"; + std::ofstream(p).close(); + auto r = util::ensure_directory(p); + REQUIRE(!r); + } + + // fails when the directory is not writable (skip when running as root, + // which bypasses permission checks) + if (::geteuid() != 0) { + auto d = *base / "readonly"; + REQUIRE(util::ensure_directory(d)); + fs::permissions(d, fs::perms::owner_read | fs::perms::owner_exec, + fs::perm_options::replace); + auto r = util::ensure_directory(d); + REQUIRE(!r); + // restore so cleanup can remove it + fs::permissions(d, fs::perms::owner_all, fs::perm_options::replace); + } +} + TEST_CASE("unsquashfs", "[fs]") { auto exe = util::exe_path(); @@ -74,7 +118,7 @@ TEST_CASE("read_single_line_file", "[fs]") { REQUIRE(!util::read_single_line_file("/wombat/soup")); REQUIRE(util::read_single_line_file("/etc/hostname")); - auto testdir = util::make_temp_dir(); + auto testdir = util::make_temp_dir().value(); // empty file { diff --git a/test/unit/mount.cpp b/test/unit/mount.cpp index 2a35cd78..3e9bab9f 100644 --- a/test/unit/mount.cpp +++ b/test/unit/mount.cpp @@ -13,7 +13,7 @@ TEST_CASE("validate_mounts", "[mount]") { // create some "fake" squashfs images // squashfs images start with the magic 4 charachters 'hsqs' - auto sqfs_root = util::make_temp_dir(); + auto sqfs_root = util::make_temp_dir().value(); auto sqfs_1 = sqfs_root / "sqfs1.squashfs"; auto sqfs_2 = sqfs_root / "sqfs2.squashfs"; auto sqfs_3 = sqfs_root / "sqfs3.squashfs"; @@ -22,11 +22,11 @@ TEST_CASE("validate_mounts", "[mount]") { std::ofstream{f} << "hsqsx"; } // create some fake squashfs - auto mount = util::make_temp_dir(); + auto mount = util::make_temp_dir().value(); auto mount_a_b = mount / "a" / "b"; auto mount_b = mount / "b"; auto mount_a = mount / "a"; - auto mount_other = util::make_temp_dir(); + auto mount_other = util::make_temp_dir().value(); // valid inputs for (auto input : { diff --git a/test/unit/oci_auth.cpp b/test/unit/oci_auth.cpp index 752957a6..aac8cd03 100644 --- a/test/unit/oci_auth.cpp +++ b/test/unit/oci_auth.cpp @@ -69,7 +69,7 @@ void write_file(const std::filesystem::path& path, std::string_view content) { } // namespace TEST_CASE("oci get_credentials reads a token file", "[oci][auth]") { - auto dir = util::make_temp_dir(); + auto dir = util::make_temp_dir().value(); auto token_file = dir / "mytoken"; // only the first line is used as the token write_file(token_file, "s3cr3t-token\nsecond line ignored\n"); @@ -104,7 +104,7 @@ TEST_CASE("oci get_credentials errors on a missing token path", } TEST_CASE("oci get_credentials reads /TOKEN", "[oci][auth]") { - auto dir = util::make_temp_dir(); + auto dir = util::make_temp_dir().value(); write_file(dir / "TOKEN", "dir-token\n"); // passing a directory reads its TOKEN entry @@ -117,14 +117,14 @@ TEST_CASE("oci get_credentials reads /TOKEN", "[oci][auth]") { TEST_CASE("oci get_credentials errors when /TOKEN is absent", "[oci][auth]") { - auto dir = util::make_temp_dir(); // empty directory, no TOKEN + auto dir = util::make_temp_dir().value(); // empty directory, no TOKEN auto c = oci::get_credentials(std::optional{"alice"}, std::optional{dir.string()}); REQUIRE_FALSE(c); } TEST_CASE("oci get_credentials falls back to the login name", "[oci][auth]") { - auto dir = util::make_temp_dir(); + auto dir = util::make_temp_dir().value(); auto token_file = dir / "tok"; write_file(token_file, "tok\n"); diff --git a/test/unit/oci_registry.cpp b/test/unit/oci_registry.cpp index 3e3faeed..c5892d81 100644 --- a/test/unit/oci_registry.cpp +++ b/test/unit/oci_registry.cpp @@ -82,7 +82,7 @@ struct zot_instance { zot_instance() { const auto zot = zot_binary(); const int port = free_port(); - state = util::make_temp_dir(); + state = util::make_temp_dir().value(); const auto cfg = state / "config.json"; { std::ofstream f{cfg}; @@ -144,7 +144,7 @@ TEST_CASE("oci registry push/pull round-trip", "[registry]") { SKIP("no zot binary available for the registry tests"); } - auto dir = util::make_temp_dir(); + auto dir = util::make_temp_dir().value(); const auto sqfs = dir / "store.squashfs"; const std::string payload = "squashfs-bytes-round-trip"; write_file(sqfs, payload); @@ -167,7 +167,7 @@ TEST_CASE("oci registry push/pull round-trip", "[registry]") { // pull the layer back; pull_squashfs self-verifies its digest internally. auto image = oci::parse_manifest(resp->body); REQUIRE(image.has_value()); - auto store = util::make_temp_dir(); + auto store = util::make_temp_dir().value(); auto pulled = oci::pull_squashfs(*c, *image, store); REQUIRE(pulled.has_value()); REQUIRE(read_file(store / "store.squashfs") == payload); @@ -179,7 +179,7 @@ TEST_CASE("oci registry attach + referrers + pull_meta", "[registry]") { SKIP("no zot binary available for the registry tests"); } - auto dir = util::make_temp_dir(); + auto dir = util::make_temp_dir().value(); const auto sqfs = dir / "store.squashfs"; write_file(sqfs, "sqfs"); const auto meta = dir / "meta"; @@ -205,7 +205,7 @@ TEST_CASE("oci registry attach + referrers + pull_meta", "[registry]") { } REQUIRE(found_meta); - auto store = util::make_temp_dir(); + auto store = util::make_temp_dir().value(); auto got = oci::pull_meta(*c, *pushed, store); REQUIRE(got.has_value()); REQUIRE(*got); @@ -219,7 +219,7 @@ TEST_CASE("oci registry copy preserves digest", "[registry]") { SKIP("no zot binary available for the registry tests"); } - auto dir = util::make_temp_dir(); + auto dir = util::make_temp_dir().value(); const auto sqfs = dir / "store.squashfs"; write_file(sqfs, "copy-me"); diff --git a/test/unit/repository.cpp b/test/unit/repository.cpp index 1bf4bfbc..854c40ca 100644 --- a/test/unit/repository.cpp +++ b/test/unit/repository.cpp @@ -97,7 +97,7 @@ TEST_CASE("find", "[repository]") { } TEST_CASE("existing", "[repository]") { - auto repo_dir = util::make_temp_dir(); + auto repo_dir = util::make_temp_dir().value(); auto repo = create_mini_repo(repo_dir); REQUIRE(repo); @@ -249,7 +249,7 @@ TEST_CASE("remove label", "[repository]") { } TEST_CASE("create disk repo", "[repository]") { - auto repo_dir = util::make_temp_dir(); + auto repo_dir = util::make_temp_dir().value(); { auto R = create_mini_repo(repo_dir); REQUIRE(R); From fff917212670ed0fd56cc8c84fc378551169438b Mon Sep 17 00:00:00 2001 From: bcumming Date: Sun, 5 Jul 2026 17:28:19 +0200 Subject: [PATCH 14/29] align token input with oras; improve error messages; polish --- .github/workflows/build_and_test.yml | 13 +++ CLAUDE.md | 10 ++- meson.build | 4 +- src/cli/copy.cpp | 5 +- src/cli/delete.cpp | 5 +- src/cli/pull.cpp | 5 +- src/cli/push.cpp | 9 +- src/cli/util.cpp | 45 ++++++++++ src/cli/util.h | 13 +++ src/oci/auth.cpp | 127 +++++++++++++++++++++++++++ src/oci/auth.h | 26 ++++++ src/oci/client.cpp | 26 +++--- src/oci/manifest.cpp | 11 +++ src/util/curl.h | 5 ++ src/util/shell.cpp | 6 +- src/util/strings.cpp | 41 +++++++++ src/util/strings.h | 7 ++ test/integration/common.bash | 37 -------- test/integration/install-zot | 2 +- test/unit/oci_auth.cpp | 110 +++++++++++++++++++++++ test/unit/oci_manifest.cpp | 16 ++++ test/unit/strings.cpp | 24 +++++ 22 files changed, 482 insertions(+), 65 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 7322e6eb..a50b4cf7 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -52,11 +52,24 @@ jobs: run: sudo ./meson/meson install -Cbuild --destdir=$PWD/staging --no-rebuild --skip-subprojects - name: create mount points for tests run: sudo mkdir /user-tools /user-environment + - name: verify zot is available for registry tests + # the zot binary is fetched at build time by test/integration/install-zot, + # which tolerates a failed download so local/offline builds still work (the + # registry suite then self-skips). in CI we require it, so live OCI + # round-trip coverage cannot silently vanish on a download hiccup. + run: | + test -x build/test/zot || { + echo "::error::zot binary missing (build/test/zot) - registry tests would silently skip" >&2 + exit 1 + } - name: unit tests run: ./meson/meson test -Cbuild --verbose unit - name: cli integration tests run: | UENV_BATS_SKIP_START=on STAGING_PATH=$PWD/staging/usr/local ./meson/meson test -Cbuild --verbose cli + - name: registry integration tests + run: | + UENV_BATS_SKIP_START=on STAGING_PATH=$PWD/staging/usr/local ./meson/meson test -Cbuild --verbose registry - name: squashfs-mount integration tests run: | UENV_BATS_SKIP_START=on STAGING_PATH=$PWD/staging/usr/local ./meson/meson test -Cbuild --verbose squashfs-mount diff --git a/CLAUDE.md b/CLAUDE.md index 265f477c..47671eea 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -115,16 +115,18 @@ a real registry, without containers or the old `oras` binary. Two helper scripts are built into `$BUILD_PATH/test` (and so are on `PATH` in all BATS tests): - `registry_ctl` — manages the lifecycle of a throwaway [zot](https://zotregistry.dev) - registry (a single static binary). Subcommands include `runtime`/`zot` (report + registry (a single static binary). Subcommands include `runtime` (report the zot binary, empty if unavailable → suite self-skips), `free-port`, `serve - STATE PORT` (backgrounds itself), `wait PORT --timeout N`, and `kill STATE`. + STATE PORT` (backgrounds itself), `wait PORT --timeout N`, `digest PORT REPO REF`, + and `kill STATE`. The zot binary is fetched at build time by `test/integration/install-zot`. - `listing_mock` — stands in for the CSCS uenv listing service (`https://uenv-list.svc.cscs.ch/list`); same `free-port`/`serve`/`wait-server`/`kill` lifecycle. Point uenv at it with `registry.listing_url` in the config. -`common.bash` wraps these with `start_registry STATE PORT` / `stop_registry` and -`start_listing LISTING_FILE PORT` / `stop_listing`. `registry.bats` writes a user +`registry.bats` drives `registry_ctl serve` / `listing_mock serve` directly from +its `setup_file` (the servers must outlive individual tests, so their state is +exported rather than held in `common.bash` helper vars). It writes a user `config.toml` whose `[registry]` block sets `url`, `default_namespace`, and `listing_url` to point at the local zot + listing_mock. The `[registry]` unit tests in `test/unit/oci_registry.cpp` cover the OCI round-trip directly by starting their diff --git a/meson.build b/meson.build index 58f5fa28..0b86930d 100644 --- a/meson.build +++ b/meson.build @@ -36,8 +36,8 @@ toml_dep = declare_dependency( dependencies: toml_base, compile_args: ['-DTOML_EXCEPTIONS=0', '-DTOML_HEADER_ONLY=0'], ) -# zlib: needed for gzip handling in the native OCI registry client that will -# replace the oras binary. Always built from the vendored wrapdb subproject +# zlib: needed for gzip handling in the native OCI registry client that +# replaced the oras binary. Always built from the vendored wrapdb subproject # (never the system copy) and statically linked, so the binary is self-contained. # Built *before* curl so its override_dependency('zlib') is in place and curl # links the same vendored static zlib rather than the system copy. diff --git a/src/cli/copy.cpp b/src/cli/copy.cpp index 5b45b2dc..59cf1e9e 100644 --- a/src/cli/copy.cpp +++ b/src/cli/copy.cpp @@ -24,6 +24,7 @@ #include "copy.h" #include "help.h" #include "terminal.h" +#include "util.h" namespace uenv { @@ -62,7 +63,9 @@ int image_copy([[maybe_unused]] const image_copy_args& args, const auto& registry_cfg = *settings.config.registry; std::optional credentials; - if (auto c = oci::get_credentials(args.username, args.token)) { + if (auto c = resolve_registry_credentials(settings.calling_environment, + registry_cfg.url, args.username, + args.token)) { credentials = *c; } else { term::error("{}", c.error()); diff --git a/src/cli/delete.cpp b/src/cli/delete.cpp index 37c54472..8211a8ca 100644 --- a/src/cli/delete.cpp +++ b/src/cli/delete.cpp @@ -21,6 +21,7 @@ #include "delete.h" #include "help.h" #include "terminal.h" +#include "util.h" namespace uenv { @@ -63,7 +64,9 @@ int image_delete([[maybe_unused]] const image_delete_args& args, const auto& artifactory_url = *registry_cfg.artifactory_url; oci::credentials credentials; - if (auto c = oci::get_credentials(args.username, args.token)) { + if (auto c = resolve_registry_credentials(settings.calling_environment, + artifactory_url, args.username, + args.token)) { if (!*c) { term::error("full credentials must be provided", c.error()); } diff --git a/src/cli/pull.cpp b/src/cli/pull.cpp index 77393eec..eda8d573 100644 --- a/src/cli/pull.cpp +++ b/src/cli/pull.cpp @@ -26,6 +26,7 @@ #include "help.h" #include "pull.h" #include "terminal.h" +#include "util.h" namespace uenv { @@ -75,7 +76,9 @@ int image_pull(const image_pull_args& args, const global_settings& settings) { const auto& registry_cfg = *settings.config.registry; std::optional credentials; - if (auto c = oci::get_credentials(args.username, args.token)) { + if (auto c = resolve_registry_credentials(settings.calling_environment, + registry_cfg.url, args.username, + args.token)) { credentials = *c; } else { term::error("{}", c.error()); diff --git a/src/cli/push.cpp b/src/cli/push.cpp index 16beeff8..ee5fd8fb 100644 --- a/src/cli/push.cpp +++ b/src/cli/push.cpp @@ -69,7 +69,9 @@ int image_push([[maybe_unused]] const image_push_args& args, const auto& registry_cfg = *settings.config.registry; std::optional credentials; - if (auto c = oci::get_credentials(args.username, args.token)) { + if (auto c = resolve_registry_credentials(settings.calling_environment, + registry_cfg.url, args.username, + args.token)) { credentials = *c; } else { term::error("{}", c.error()); @@ -217,8 +219,9 @@ std::string image_push_footer() { help::block{xmpl, "use a token for the registry"}, help::block{code, "uenv image push --token=/opt/cscs/uenv/tokens/vasp6 \\"}, help::block{code, " ./store.squashfs prgenv-gnu/24.11:v3%gh200@daint"}, - help::block{note, "this is required if you have not configured ~/.docker/config.json" }, - help::block{none, "with the token for the registry." }, + help::block{note, "credentials are resolved in order: --token, then the" }, + help::block{none, "uenv token store $XDG_CONFIG_HOME/uenv/tokens/," }, + help::block{none, "then ~/.docker/config.json." }, // clang-format on }; diff --git a/src/cli/util.cpp b/src/cli/util.cpp index c3938c15..9940ebfa 100644 --- a/src/cli/util.cpp +++ b/src/cli/util.cpp @@ -11,12 +11,57 @@ #include #include +#include +#include #include #include "util.h" namespace uenv { +util::expected, std::string> +resolve_registry_credentials(const envvars::state& env, + const std::string& registry_url, + std::optional username, + std::optional token) { + namespace fs = std::filesystem; + + // the credential store is keyed by the bare registry host[:port]; derive it + // from the configured registry url. + auto loc = oci::split_registry(registry_url); + if (!loc) { + return util::unexpected{ + fmt::format("invalid registry url: {}", loc.error().message())}; + } + std::string host = loc->base; + if (auto p = host.find("://"); p != std::string::npos) { + host.erase(0, p + 3); + } + + oci::credential_sources sources; + if (token) { + sources.explicit_token = fs::path{*token}; + } + sources.username = std::move(username); + + // the uenv token store: $XDG_CONFIG_HOME/uenv/tokens or ~/.config/uenv/tokens. + if (auto xdg = env.get("XDG_CONFIG_HOME")) { + sources.uenv_token_dir = fs::path{*xdg} / "uenv" / "tokens"; + } else if (auto home = env.get("HOME")) { + sources.uenv_token_dir = fs::path{*home} / ".config" / "uenv" / "tokens"; + } + + // docker config.json fallback: $DOCKER_CONFIG/config.json or + // ~/.docker/config.json (a missing file is handled gracefully downstream). + if (auto dcfg = env.get("DOCKER_CONFIG")) { + sources.docker_config = fs::path{*dcfg} / "config.json"; + } else if (auto home = env.get("HOME")) { + sources.docker_config = fs::path{*home} / ".docker" / "config.json"; + } + + return oci::resolve_credentials(host, sources); +} + // returns true if squashfs-mount 9 or later is detected in PATH. // this is used for a compatibility layer bool sqfs_mount_v9(const envvars::state& calling_environment) { diff --git a/src/cli/util.h b/src/cli/util.h index f2142393..a9189003 100644 --- a/src/cli/util.h +++ b/src/cli/util.h @@ -10,8 +10,21 @@ #include #include +#include + namespace uenv { +// Resolve registry credentials for the registry named by `registry_url`, trying +// (in order) an explicit --token, the uenv token store +// ($XDG_CONFIG_HOME/uenv/tokens/), and ~/.docker/config.json. Returns +// std::nullopt for anonymous access. This assembles the uenv/environment paths +// and delegates to oci::resolve_credentials. +util::expected, std::string> +resolve_registry_credentials(const envvars::state& env, + const std::string& registry_url, + std::optional username, + std::optional token); + struct squashfs_image { // the absolute path of the squashfs file std::filesystem::path sqfs; diff --git a/src/oci/auth.cpp b/src/oci/auth.cpp index 52a00c77..e2653523 100644 --- a/src/oci/auth.cpp +++ b/src/oci/auth.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -200,4 +201,130 @@ get_credentials(std::optional username, "provide a username with --username for the --token."}; } +namespace { + +// build credentials from a token string, taking the username from `username` +// or, failing that, the OS login name. +util::expected +creds_from_token(std::string token, const std::optional& username) { + if (username) { + return credentials{.username = *username, .password = std::move(token)}; + } + if (auto name = getlogin()) { + return credentials{.username = name, .password = std::move(token)}; + } + return util::unexpected{"provide a username with --username for the token."}; +} + +// normalise a docker config.json `auths` key (or a registry host) down to a bare +// host[:port] for comparison: drop any scheme and any trailing path. +std::string normalise_host(std::string_view key) { + auto s = key; + if (auto p = s.find("://"); p != std::string_view::npos) { + s.remove_prefix(p + 3); + } + if (auto p = s.find('/'); p != std::string_view::npos) { + s = s.substr(0, p); + } + return std::string{s}; +} + +// look up credentials for `host` in a docker config.json file. Returns nullopt +// when the file has no matching, usable entry. +util::expected, std::string> +creds_from_docker_config(const std::filesystem::path& cfg, + std::string_view host) { + std::ifstream in(cfg, std::ios::binary); + if (!in) { + return std::nullopt; // no readable config: not an error, just no creds + } + std::string body((std::istreambuf_iterator(in)), + std::istreambuf_iterator()); + auto j = nlohmann::json::parse(body, nullptr, /*allow_exceptions=*/false); + if (j.is_discarded() || !j.is_object()) { + return util::unexpected{ + fmt::format("could not parse docker config {}", cfg.string())}; + } + auto auths = j.find("auths"); + if (auths == j.end() || !auths->is_object()) { + return std::nullopt; + } + const std::string want = normalise_host(host); + for (const auto& [key, entry] : auths->items()) { + if (normalise_host(key) != want || !entry.is_object()) { + continue; + } + auto a = entry.find("auth"); + if (a == entry.end() || !a->is_string()) { + // credsStore / credHelpers / identitytoken entries need an external + // helper we do not implement. + spdlog::warn("docker config entry for {} has no 'auth' field " + "(credential helpers are not supported)", + want); + return std::nullopt; + } + auto decoded = util::base64_decode(a->get()); + if (!decoded) { + return util::unexpected{fmt::format( + "invalid base64 in docker config auth for {}", want)}; + } + auto colon = decoded->find(':'); + if (colon == std::string::npos) { + return util::unexpected{fmt::format( + "malformed docker config auth for {} (expected user:pass)", + want)}; + } + return credentials{.username = decoded->substr(0, colon), + .password = decoded->substr(colon + 1)}; + } + return std::nullopt; +} + +} // namespace + +util::expected, std::string> +resolve_credentials(std::string_view registry_host, + const credential_sources& src) { + // 1. an explicit --token wins. + if (src.explicit_token) { + return get_credentials(src.username, src.explicit_token->string()); + } + + // 2. the uenv token store: /. + if (src.uenv_token_dir) { + auto token_path = *src.uenv_token_dir / std::string{registry_host}; + if (util::file_access_level(token_path) >= util::file_level::readonly) { + // warn (but do not fail) if the token file is readable by group or + // others, since it holds a secret. + std::error_code ec; + auto perms = std::filesystem::status(token_path, ec).permissions(); + using std::filesystem::perms; + if (!ec && (perms & (perms::group_read | perms::others_read)) != + perms::none) { + spdlog::warn("token file {} is readable by group/others; " + "consider `chmod 600`", + token_path.string()); + } + auto token = util::read_single_line_file(token_path); + if (!token) { + return util::unexpected{fmt::format( + "unable to read a token from {}", token_path.string())}; + } + auto creds = creds_from_token(*token, src.username); + if (!creds) { + return util::unexpected{creds.error()}; + } + return *creds; + } + } + + // 3. docker config.json (interop fallback). + if (src.docker_config) { + return creds_from_docker_config(*src.docker_config, registry_host); + } + + // 4. no credentials found: anonymous access. + return std::nullopt; +} + } // namespace oci diff --git a/src/oci/auth.h b/src/oci/auth.h index 201db27d..c60d6e11 100644 --- a/src/oci/auth.h +++ b/src/oci/auth.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -63,6 +64,31 @@ util::expected, std::string> get_credentials(std::optional username, std::optional token); +// The candidate sources consulted by `resolve_credentials`, in precedence order. +// The CLI populates these (it owns the uenv-specific and environment-specific +// paths); this keeps `src/oci` free of any dependency on `src/uenv`. +struct credential_sources { + // an explicit --token path (a file, or a directory holding a `TOKEN` file). + std::optional explicit_token; + // an explicit --username; when unset the OS login name is used. + std::optional username; + // the uenv token store directory, e.g. $XDG_CONFIG_HOME/uenv/tokens. The + // token for a registry is read from /. + std::optional uenv_token_dir; + // a docker config.json path (e.g. ~/.docker/config.json) for interop. + std::optional docker_config; +}; + +// Resolve credentials for `registry_host` from `src`, trying, in order: +// 1. an explicit --token (delegates to get_credentials), +// 2. the uenv token store /, +// 3. the docker config.json `auths[].auth` entry. +// Returns std::nullopt when no credentials are found (anonymous access), or an +// error string when a source was present but could not be used. +util::expected, std::string> +resolve_credentials(std::string_view registry_host, + const credential_sources& src); + } // namespace oci #include diff --git a/src/oci/client.cpp b/src/oci/client.cpp index b4d52566..f7bcc700 100644 --- a/src/oci/client.cpp +++ b/src/oci/client.cpp @@ -257,7 +257,8 @@ util::expected client::get_blob(const digest& d) { } if (resp->status != 200) { return util::unexpected{fmt::format( - "failed to fetch blob {} (status {})", d.string(), resp->status)}; + "failed to fetch blob {} (status {}): {}", d.string(), resp->status, + util::curl::http_message(resp->status))}; } return resp->body; } @@ -308,7 +309,8 @@ client::get_blob_to_file( } if (resp->status != 200) { return util::unexpected{fmt::format( - "failed to fetch blob {} (status {})", d.string(), resp->status)}; + "failed to fetch blob {} (status {}): {}", d.string(), resp->status, + util::curl::http_message(resp->status))}; } if (verify) { @@ -344,8 +346,8 @@ client::get_manifest(const reference& ref) { } if (resp->status != 200) { return util::unexpected{fmt::format( - "failed to fetch manifest {} (status {})", ref.string(), - resp->status)}; + "failed to fetch manifest {} (status {}): {}", ref.string(), + resp->status, util::curl::http_message(resp->status))}; } manifest_response m; m.body = std::move(resp->body); @@ -398,8 +400,8 @@ client::referrers(const digest& d) { } if (resp->status != 200) { return util::unexpected{fmt::format( - "failed to fetch referrers of {} (status {})", d.string(), - resp->status)}; + "failed to fetch referrers of {} (status {}): {}", d.string(), + resp->status, util::curl::http_message(resp->status))}; } auto refs = impl::parse_referrers(resp->body); if (!refs) { @@ -427,8 +429,8 @@ do_put_blob(const std::string& registry_url, const std::string& uploads, } if (opened->status != 202) { return util::unexpected{fmt::format( - "failed to open upload session (status {}): {}", opened->status, - opened->body)}; + "failed to open upload session (status {}): {} {}", opened->status, + util::curl::http_message(opened->status), opened->body)}; } auto location = opened->headers.get("location"); if (!location) { @@ -451,8 +453,8 @@ do_put_blob(const std::string& registry_url, const std::string& uploads, } if (done->status != 201) { return util::unexpected{fmt::format( - "failed to upload blob {} (status {}): {}", digest, done->status, - done->body)}; + "failed to upload blob {} (status {}): {} {}", digest, done->status, + util::curl::http_message(done->status), done->body)}; } return {}; } @@ -544,8 +546,8 @@ client::put_manifest(const reference& ref, const std::string& body, } if (resp->status != 201) { return util::unexpected{fmt::format( - "failed to put manifest {} (status {}): {}", ref.string(), - resp->status, resp->body)}; + "failed to put manifest {} (status {}): {} {}", ref.string(), + resp->status, util::curl::http_message(resp->status), resp->body)}; } return {}; } diff --git a/src/oci/manifest.cpp b/src/oci/manifest.cpp index 5aaab601..f89730ac 100644 --- a/src/oci/manifest.cpp +++ b/src/oci/manifest.cpp @@ -171,6 +171,17 @@ util::expected parse_manifest(std::string_view body) { m.media_type = j.value("mediaType", std::string{media_type_manifest}); m.artifact_type = j.value("artifactType", std::string{}); + // reject multi-arch image indexes / manifest lists: they carry a `manifests` + // array of per-platform descriptors instead of `layers`, so treating one as + // an image manifest would silently yield an empty layer set. uenv only ever + // deals with single-platform artifacts. + if (m.media_type == media_type_index || + m.media_type == "application/vnd.docker.distribution.manifest.list.v2+json" || + j.contains("manifests")) { + return util::unexpected{ + "multi-arch image indexes (manifest lists) are not supported"}; + } + if (auto c = j.find("config"); c != j.end() && c->is_object()) { auto d = parse_descriptor(*c); if (!d) { diff --git a/src/util/curl.h b/src/util/curl.h index 633e8cac..ec05c8e6 100644 --- a/src/util/curl.h +++ b/src/util/curl.h @@ -99,6 +99,11 @@ struct response { expected perform(const request& req); +// a friendly, user-facing explanation for an HTTP status code (e.g. a hint that +// 403 means invalid credentials). falls back to a generic message. intended to +// enrich the terse "status N" errors reported by the OCI client. +std::string http_message(long code); + // parse a single raw HTTP header line ("Name: value\r\n") into a headers map. // status lines and blank separators are ignored. exposed for unit testing. void parse_header_line(headers& h, std::string_view line); diff --git a/src/util/shell.cpp b/src/util/shell.cpp index 74c62ece..0c317d48 100644 --- a/src/util/shell.cpp +++ b/src/util/shell.cpp @@ -88,10 +88,10 @@ exec_error exec(const std::vector& args, char* const envp[]) { } argv.push_back(nullptr); - // WARNING: do not log all arguments to the command that is being executbd + // WARNING: do not log all arguments to the command that is being executed // because it might leak secrets: it is the responsibility of - // callers to log the call appropriately, for example the oras - // wrappers that remove secrets before printing. + // callers to log the call appropriately, removing any secrets + // before printing. int r; if (envp == nullptr) { spdlog::info("execvp({})", args[0]); diff --git a/src/util/strings.cpp b/src/util/strings.cpp index ea5aab2a..ddc1d085 100644 --- a/src/util/strings.cpp +++ b/src/util/strings.cpp @@ -109,4 +109,45 @@ bool is_sha(const std::string& str) { return false; } +std::optional base64_decode(std::string_view input) { + auto value = [](unsigned char c) -> int { + if (c >= 'A' && c <= 'Z') { + return c - 'A'; + } + if (c >= 'a' && c <= 'z') { + return c - 'a' + 26; + } + if (c >= '0' && c <= '9') { + return c - '0' + 52; + } + if (c == '+') { + return 62; + } + if (c == '/') { + return 63; + } + return -1; + }; + + std::string out; + int buffer = 0; + int bits = 0; + for (unsigned char c : input) { + if (c == '=' || std::isspace(c)) { + continue; // padding / whitespace: ignore + } + int v = value(c); + if (v < 0) { + return std::nullopt; // invalid character + } + buffer = (buffer << 6) | v; + bits += 6; + if (bits >= 8) { + bits -= 8; + out.push_back(static_cast((buffer >> bits) & 0xFF)); + } + } + return out; +} + } // namespace util diff --git a/src/util/strings.h b/src/util/strings.h index 3871a78d..ba63a0ff 100644 --- a/src/util/strings.h +++ b/src/util/strings.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -48,4 +49,10 @@ std::string to_lower(std::string_view input); std::string join(std::string_view joiner, const std::vector& list); bool is_sha(const std::string& str); + +// decode a standard base64 string (RFC 4648). Padding '=' and embedded +// whitespace are ignored. Returns nullopt if the input contains a character +// outside the base64 alphabet. Note: this is NOT the URL-safe ('-'/'_') +// alphabet. +std::optional base64_decode(std::string_view input); } // namespace util diff --git a/test/integration/common.bash b/test/integration/common.bash index c8bb4cfe..c8bf1d91 100755 --- a/test/integration/common.bash +++ b/test/integration/common.bash @@ -38,43 +38,6 @@ function wait_elastic_post() { elastic_mock wait "$1" --timeout "${2:-10}" --count "${3:-1}" } -# ──────────────────────── registry (zot) + listing helpers ─────────────────── - -REGISTRY_STATE="" -LISTING_MOCK_PID="" - -# start_registry STATE PORT: run a throwaway zot registry on 127.0.0.1:PORT. -function start_registry() { - REGISTRY_STATE="$1" - local port="$2" - registry_ctl serve "$REGISTRY_STATE" "$port" & - registry_ctl wait "$port" --timeout 30 -} - -function stop_registry() { - if [[ -n "${REGISTRY_STATE:-}" ]]; then - registry_ctl kill "$REGISTRY_STATE" 2>/dev/null || true - REGISTRY_STATE="" - fi -} - -# start_listing LISTING_FILE PORT: run the listing_mock server on 127.0.0.1:PORT. -function start_listing() { - local listing_file="$1" - local port="$2" - listing_mock serve "$listing_file" "$port" & - LISTING_MOCK_PID=$! - listing_mock wait-server "$port" --timeout 5 -} - -function stop_listing() { - if [[ -n "${LISTING_MOCK_PID:-}" ]]; then - kill "$LISTING_MOCK_PID" 2>/dev/null || true - wait "$LISTING_MOCK_PID" 2>/dev/null || true - LISTING_MOCK_PID="" - fi -} - function run_srun_unchecked() { log "+ srun $@" run srun -n1 --oversubscribe "$@" diff --git a/test/integration/install-zot b/test/integration/install-zot index 5b8c2ba5..36c441ff 100755 --- a/test/integration/install-zot +++ b/test/integration/install-zot @@ -21,7 +21,7 @@ url="https://github.com/project-zot/zot/releases/download/${version}/zot-linux-$ # download to a temp file then move into place, so a failed/partial download # does not leave a broken "zot" that looks complete to ninja. tmp="${output}.download" -if curl -fL "$url" -o "$tmp" 2>/dev/null; then +if curl -fL --retry 3 --retry-delay 2 "$url" -o "$tmp" 2>/dev/null; then chmod +x "$tmp" mv "$tmp" "$output" else diff --git a/test/unit/oci_auth.cpp b/test/unit/oci_auth.cpp index aac8cd03..56a39942 100644 --- a/test/unit/oci_auth.cpp +++ b/test/unit/oci_auth.cpp @@ -149,3 +149,113 @@ TEST_CASE("oci credentials formatter redacts the password", "[oci][auth]") { REQUIRE(s.find("secret") == std::string::npos); // password must not leak REQUIRE(s.find("XXXXXX") != std::string::npos); // 6 chars -> 6 'X' } + +TEST_CASE("oci resolve_credentials precedence", "[oci][auth]") { + const std::string host = "reg.example.com"; + + // an explicit --token wins over everything else. + SECTION("explicit token") { + auto dir = util::make_temp_dir().value(); + auto tok = dir / "tok"; + write_file(tok, "explicit-tok\n"); + oci::credential_sources src; + src.explicit_token = tok; + src.username = "alice"; + src.uenv_token_dir = dir; // present but must be ignored + auto c = oci::resolve_credentials(host, src); + REQUIRE(c); + REQUIRE(c->has_value()); + REQUIRE((*c)->username == "alice"); + REQUIRE((*c)->password == "explicit-tok"); + } + + // the uenv token store: /. + SECTION("uenv token store") { + auto dir = util::make_temp_dir().value(); + write_file(dir / host, "store-tok\n"); + oci::credential_sources src; + src.username = "bob"; + src.uenv_token_dir = dir; + auto c = oci::resolve_credentials(host, src); + REQUIRE(c); + REQUIRE(c->has_value()); + REQUIRE((*c)->username == "bob"); + REQUIRE((*c)->password == "store-tok"); + } + + // docker config.json auths[host].auth is base64(user:pass). + SECTION("docker config fallback") { + auto dir = util::make_temp_dir().value(); + auto cfg = dir / "config.json"; + // echo -n carol:pw | base64 -> Y2Fyb2w6cHc= + write_file(cfg, R"({"auths":{"reg.example.com":{"auth":"Y2Fyb2w6cHc="}}})"); + oci::credential_sources src; + src.docker_config = cfg; + auto c = oci::resolve_credentials(host, src); + REQUIRE(c); + REQUIRE(c->has_value()); + REQUIRE((*c)->username == "carol"); + REQUIRE((*c)->password == "pw"); + } + + // docker keys may carry a scheme/path; matching is on the bare host. + SECTION("docker config host normalisation") { + auto dir = util::make_temp_dir().value(); + auto cfg = dir / "config.json"; + write_file(cfg, + R"({"auths":{"https://reg.example.com/v2/":{"auth":"Y2Fyb2w6cHc="}}})"); + oci::credential_sources src; + src.docker_config = cfg; + auto c = oci::resolve_credentials(host, src); + REQUIRE(c); + REQUIRE(c->has_value()); + REQUIRE((*c)->username == "carol"); + } + + // no matching source -> anonymous (nullopt), not an error. + SECTION("no source is anonymous") { + auto dir = util::make_temp_dir().value(); // empty store, no host file + oci::credential_sources src; + src.uenv_token_dir = dir; + src.docker_config = dir / "does-not-exist.json"; + auto c = oci::resolve_credentials(host, src); + REQUIRE(c); + REQUIRE_FALSE(c->has_value()); + } + + // a config entry with no host match -> anonymous. + SECTION("docker config host miss") { + auto dir = util::make_temp_dir().value(); + auto cfg = dir / "config.json"; + write_file(cfg, R"({"auths":{"other.example.com":{"auth":"Y2Fyb2w6cHc="}}})"); + oci::credential_sources src; + src.docker_config = cfg; + auto c = oci::resolve_credentials(host, src); + REQUIRE(c); + REQUIRE_FALSE(c->has_value()); + } + + // a credsStore-only entry (no `auth`) is skipped, not an error. + SECTION("docker config credsStore entry skipped") { + auto dir = util::make_temp_dir().value(); + auto cfg = dir / "config.json"; + write_file(cfg, + R"({"auths":{"reg.example.com":{}},"credsStore":"desktop"})"); + oci::credential_sources src; + src.docker_config = cfg; + auto c = oci::resolve_credentials(host, src); + REQUIRE(c); + REQUIRE_FALSE(c->has_value()); + } + + // malformed base64 in the auth field is an error. + SECTION("docker config malformed auth") { + auto dir = util::make_temp_dir().value(); + auto cfg = dir / "config.json"; + write_file(cfg, R"({"auths":{"reg.example.com":{"auth":"@@@@"}}})"); + oci::credential_sources src; + src.docker_config = cfg; + auto c = oci::resolve_credentials(host, src); + REQUIRE_FALSE(c); + } +} diff --git a/test/unit/oci_manifest.cpp b/test/unit/oci_manifest.cpp index 6d4679e3..f6b355a0 100644 --- a/test/unit/oci_manifest.cpp +++ b/test/unit/oci_manifest.cpp @@ -126,6 +126,22 @@ TEST_CASE("parse_manifest rejects malformed", "[oci][manifest]") { .has_value()); } +TEST_CASE("parse_manifest rejects image indexes", "[oci][manifest]") { + // by media type + REQUIRE_FALSE( + parse_manifest( + R"({"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[]})") + .has_value()); + // by docker manifest-list media type + REQUIRE_FALSE( + parse_manifest( + R"({"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.list.v2+json"})") + .has_value()); + // by the presence of a `manifests` array even without an index media type + REQUIRE_FALSE( + parse_manifest(R"({"schemaVersion":2,"manifests":[]})").has_value()); +} + TEST_CASE("serialize image index", "[oci][manifest]") { descriptor d{.media_type = std::string{media_type_manifest}, .digest = digest::sha256(image_manifest_hex), diff --git a/test/unit/strings.cpp b/test/unit/strings.cpp index ea9522b7..d281d954 100644 --- a/test/unit/strings.cpp +++ b/test/unit/strings.cpp @@ -43,3 +43,27 @@ TEST_CASE("split", "[strings]") { REQUIRE(util::split("a,b,c", ',', true) == v{"a", "b", "c"}); REQUIRE(util::split("a,b,,c", ',', true) == v{"a", "b", "c"}); } + +TEST_CASE("base64_decode", "[strings]") { + auto dec = [](std::string_view s) { return util::base64_decode(s); }; + + // RFC 4648 test vectors + REQUIRE(dec("") == std::optional{""}); + REQUIRE(dec("Zg==") == std::optional{"f"}); + REQUIRE(dec("Zm8=") == std::optional{"fo"}); + REQUIRE(dec("Zm9v") == std::optional{"foo"}); + REQUIRE(dec("Zm9vYg==") == std::optional{"foob"}); + REQUIRE(dec("Zm9vYmE=") == std::optional{"fooba"}); + REQUIRE(dec("Zm9vYmFy") == std::optional{"foobar"}); + + // a docker-style user:pass token + REQUIRE(dec("YWxpY2U6czNjcmV0") == std::optional{"alice:s3cret"}); + + // padding is optional; embedded whitespace/newlines are ignored + REQUIRE(dec("Zm9v\n") == std::optional{"foo"}); + REQUIRE(dec("Zg") == std::optional{"f"}); + + // characters outside the standard alphabet are rejected + REQUIRE(dec("****") == std::nullopt); + REQUIRE(dec("ab_c") == std::nullopt); // url-safe alphabet is not accepted +} From fc5919bac53eeec2eb073b682501f5cad6a839c0 Mon Sep 17 00:00:00 2001 From: bcumming Date: Sun, 5 Jul 2026 22:19:48 +0200 Subject: [PATCH 15/29] fix race reporting download progress --- src/cli/pull.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/cli/pull.cpp b/src/cli/pull.cpp index eda8d573..a6a16371 100644 --- a/src/cli/pull.cpp +++ b/src/cli/pull.cpp @@ -1,5 +1,6 @@ // vim: ts=4 sts=4 sw=4 et +#include #include #include #include @@ -245,7 +246,9 @@ int image_pull(const image_pull_args& args, const global_settings& settings) { if (pull_sqfs) { namespace bk = barkeep; - std::size_t downloaded_mb{0u}; + // atomic: written by curl's progress callback on the main + // thread, read concurrently by barkeep's display thread. + std::atomic downloaded_mb{0u}; // round up so total_mb is never zero std::size_t total_mb{(record.size_byte + (1024 * 1024 - 1)) / (1024 * 1024)}; From 904fed03e96f626e09d6dee699a165f2f74b889f Mon Sep 17 00:00:00 2001 From: bcumming Date: Sun, 5 Jul 2026 22:36:44 +0200 Subject: [PATCH 16/29] apply clang format --- src/cli/copy.cpp | 2 +- src/cli/delete.cpp | 7 ++-- src/cli/pull.cpp | 25 +++++------ src/cli/push.cpp | 14 +++---- src/cli/util.cpp | 6 ++- src/oci/auth.cpp | 28 +++++++------ src/oci/auth.h | 7 ++-- src/oci/client.cpp | 62 +++++++++++++-------------- src/oci/client.h | 18 ++++---- src/oci/digest.h | 11 ++--- src/oci/manifest.cpp | 26 ++++++------ src/oci/manifest.h | 14 +++---- src/oci/parse.cpp | 52 ++++++++++++----------- src/oci/parse.h | 21 +++++----- src/oci/pull.cpp | 3 +- src/oci/pull.h | 12 +++--- src/oci/push.cpp | 85 ++++++++++++++++++++------------------ src/oci/reference.h | 4 +- src/uenv/parse.cpp | 3 +- src/uenv/settings.cpp | 5 ++- src/util/curl.cpp | 43 ++++++++++--------- src/util/curl.h | 10 +++-- src/util/fs.cpp | 11 +++-- src/util/sha256.cpp | 3 +- test/unit/curl.cpp | 10 +++-- test/unit/oci_auth.cpp | 50 +++++++++++----------- test/unit/oci_client.cpp | 15 +++---- test/unit/oci_manifest.cpp | 15 ++++--- test/unit/oci_parse.cpp | 6 +-- test/unit/oci_registry.cpp | 24 ++++++----- test/unit/sha256.cpp | 12 +++--- test/unit/strings.cpp | 3 +- 32 files changed, 317 insertions(+), 290 deletions(-) diff --git a/src/cli/copy.cpp b/src/cli/copy.cpp index 59cf1e9e..bf14e99f 100644 --- a/src/cli/copy.cpp +++ b/src/cli/copy.cpp @@ -9,10 +9,10 @@ #include #include -#include #include #include #include +#include #include #include #include diff --git a/src/cli/delete.cpp b/src/cli/delete.cpp index 8211a8ca..98441da9 100644 --- a/src/cli/delete.cpp +++ b/src/cli/delete.cpp @@ -8,8 +8,8 @@ #include #include -#include #include +#include #include #include #include @@ -128,9 +128,8 @@ int image_delete([[maybe_unused]] const image_delete_args& args, record.system, record.uarch, record.name, record.version, record.tag); - if (auto result = - util::curl::del(url, credentials.username, - credentials.password); + if (auto result = util::curl::del(url, credentials.username, + credentials.password); !result) { term::error("unable to delete uenv: {}", result.error().message); return 1; diff --git a/src/cli/pull.cpp b/src/cli/pull.cpp index a6a16371..46d52b7a 100644 --- a/src/cli/pull.cpp +++ b/src/cli/pull.cpp @@ -12,9 +12,9 @@ #include #include -#include #include #include +#include #include #include #include @@ -209,8 +209,8 @@ int image_pull(const image_pull_args& args, const global_settings& settings) { const auto manifest_ref = oci::reference::digest(image_digest); try { - // the image manifest is needed for the squashfs layer; fetch + parse - // once. + // the image manifest is needed for the squashfs layer; fetch + + // parse once. auto response = client->get_manifest(manifest_ref); if (!response) { term::error("unable to fetch the image manifest:\n{}", @@ -225,8 +225,7 @@ int image_pull(const image_pull_args& args, const global_settings& settings) { } if (pull_meta) { - auto found = - oci::pull_meta(*client, image_digest, paths.store); + auto found = oci::pull_meta(*client, image_digest, paths.store); if (!found) { term::error("unable to pull meta data.\n{}", found.error()); return 1; @@ -256,7 +255,8 @@ int image_pull(const image_pull_args& args, const global_settings& settings) { &downloaded_mb, { .total = total_mb, - .message = fmt::format("pulling {}", record.id.string()), + .message = + fmt::format("pulling {}", record.id.string()), .speed = 0.1, .speed_unit = "MB/s", .style = color::use_color() @@ -271,14 +271,14 @@ int image_pull(const image_pull_args& args, const global_settings& settings) { }; // util::signal_raised() consumes (resets) the flag, so it must // be checked exactly once. latch the result here in the abort - // predicate; the post-download check then reads the latch rather - // than calling signal_raised() again (which would see false and - // skip the cleanup, leaving a partial download behind). + // predicate; the post-download check then reads the latch + // rather than calling signal_raised() again (which would see + // false and skip the cleanup, leaving a partial download + // behind). bool aborted = false; util::set_signal_catcher(); auto result = oci::pull_squashfs( - *client, *manifest, paths.store, progress, - [&aborted]() { + *client, *manifest, paths.store, progress, [&aborted]() { aborted = aborted || util::signal_raised(); return aborted; }); @@ -292,7 +292,8 @@ int image_pull(const image_pull_args& args, const global_settings& settings) { // a Ctrl-C during the download aborts the transfer; surface // it as a signal so the cleanup below runs. if (aborted) { - throw util::signal_exception(util::last_signal_raised()); + throw util::signal_exception( + util::last_signal_raised()); } term::error("unable to pull uenv.\n{}", result.error()); return 1; diff --git a/src/cli/push.cpp b/src/cli/push.cpp index ee5fd8fb..a68e9ad8 100644 --- a/src/cli/push.cpp +++ b/src/cli/push.cpp @@ -10,11 +10,11 @@ #include #include -#include #include #include #include #include +#include #include #include #include @@ -138,8 +138,8 @@ int image_push([[maybe_unused]] const image_push_args& args, return 1; } const auto& L = dst_label.label; - const auto repository = oci::repository_path( - loc->prefix, nspace, *L.system, *L.uarch, *L.name, *L.version); + const auto repository = oci::repository_path(loc->prefix, nspace, *L.system, + *L.uarch, *L.name, *L.version); spdlog::debug("oci push: registry={} repository={}", loc->base, repository); auto client = oci::client::create(loc->base, repository, credentials); @@ -151,8 +151,8 @@ int image_push([[maybe_unused]] const image_push_args& args, namespace bk = barkeep; // Push the SquashFS image. Ctrl-C is left with its default behaviour: the - // upload session is not committed until the manifest is PUT, so an interrupt - // leaves nothing referencing a partial blob. + // upload session is not committed until the manifest is PUT, so an + // interrupt leaves nothing referencing a partial blob. std::optional image_digest; { auto spinner = bk::Animation({ @@ -161,8 +161,8 @@ int image_push([[maybe_unused]] const image_push_args& args, .style = bk::Ellipsis, .no_tty = !isatty(fileno(stdout)), }); - auto push_result = - oci::push_squashfs(*client, sqfs->sqfs, oci::reference::tag(*L.tag)); + auto push_result = oci::push_squashfs(*client, sqfs->sqfs, + oci::reference::tag(*L.tag)); spinner->done(); if (!push_result) { term::error("unable to push uenv.\n{}", push_result.error()); diff --git a/src/cli/util.cpp b/src/cli/util.cpp index 9940ebfa..9b62b084 100644 --- a/src/cli/util.cpp +++ b/src/cli/util.cpp @@ -44,11 +44,13 @@ resolve_registry_credentials(const envvars::state& env, } sources.username = std::move(username); - // the uenv token store: $XDG_CONFIG_HOME/uenv/tokens or ~/.config/uenv/tokens. + // the uenv token store: $XDG_CONFIG_HOME/uenv/tokens or + // ~/.config/uenv/tokens. if (auto xdg = env.get("XDG_CONFIG_HOME")) { sources.uenv_token_dir = fs::path{*xdg} / "uenv" / "tokens"; } else if (auto home = env.get("HOME")) { - sources.uenv_token_dir = fs::path{*home} / ".config" / "uenv" / "tokens"; + sources.uenv_token_dir = + fs::path{*home} / ".config" / "uenv" / "tokens"; } // docker config.json fallback: $DOCKER_CONFIG/config.json or diff --git a/src/oci/auth.cpp b/src/oci/auth.cpp index e2653523..2dfd16e8 100644 --- a/src/oci/auth.cpp +++ b/src/oci/auth.cpp @@ -12,12 +12,12 @@ #include #include +#include +#include #include #include #include #include -#include -#include namespace oci { @@ -91,8 +91,8 @@ discover_challenge(const std::string& registry_url) { return std::nullopt; } if (resp->status != 401) { - return util::unexpected{fmt::format( - "unexpected status {} probing {}", resp->status, req.url)}; + return util::unexpected{fmt::format("unexpected status {} probing {}", + resp->status, req.url)}; } auto header = resp->headers.get("www-authenticate"); if (!header) { @@ -101,9 +101,9 @@ discover_challenge(const std::string& registry_url) { } auto challenge = parse_bearer_challenge(*header); if (!challenge) { - return util::unexpected{fmt::format( - "could not parse WWW-Authenticate header '{}': {}", *header, - challenge.error().message())}; + return util::unexpected{ + fmt::format("could not parse WWW-Authenticate header '{}': {}", + *header, challenge.error().message())}; } return *challenge; } @@ -122,8 +122,8 @@ fetch_token(const bearer_challenge& challenge, auto resp = util::curl::perform(req); if (!resp) { - return util::unexpected{fmt::format("token request failed: {}", - resp.error().message)}; + return util::unexpected{ + fmt::format("token request failed: {}", resp.error().message)}; } if (resp->status != 200) { return util::unexpected{ @@ -206,18 +206,20 @@ namespace { // build credentials from a token string, taking the username from `username` // or, failing that, the OS login name. util::expected -creds_from_token(std::string token, const std::optional& username) { +creds_from_token(std::string token, + const std::optional& username) { if (username) { return credentials{.username = *username, .password = std::move(token)}; } if (auto name = getlogin()) { return credentials{.username = name, .password = std::move(token)}; } - return util::unexpected{"provide a username with --username for the token."}; + return util::unexpected{ + "provide a username with --username for the token."}; } -// normalise a docker config.json `auths` key (or a registry host) down to a bare -// host[:port] for comparison: drop any scheme and any trailing path. +// normalise a docker config.json `auths` key (or a registry host) down to a +// bare host[:port] for comparison: drop any scheme and any trailing path. std::string normalise_host(std::string_view key) { auto s = key; if (auto p = s.find("://"); p != std::string_view::npos) { diff --git a/src/oci/auth.h b/src/oci/auth.h index c60d6e11..565f30c5 100644 --- a/src/oci/auth.h +++ b/src/oci/auth.h @@ -64,9 +64,10 @@ util::expected, std::string> get_credentials(std::optional username, std::optional token); -// The candidate sources consulted by `resolve_credentials`, in precedence order. -// The CLI populates these (it owns the uenv-specific and environment-specific -// paths); this keeps `src/oci` free of any dependency on `src/uenv`. +// The candidate sources consulted by `resolve_credentials`, in precedence +// order. The CLI populates these (it owns the uenv-specific and +// environment-specific paths); this keeps `src/oci` free of any dependency on +// `src/uenv`. struct credential_sources { // an explicit --token path (a file, or a directory holding a `TOKEN` file). std::optional explicit_token; diff --git a/src/oci/client.cpp b/src/oci/client.cpp index f7bcc700..4c5a71d3 100644 --- a/src/oci/client.cpp +++ b/src/oci/client.cpp @@ -11,12 +11,12 @@ #include #include +#include +#include #include #include #include #include -#include -#include namespace oci { @@ -55,7 +55,8 @@ std::string resolve_upload_url(std::string_view registry_url, std::string_view location, std::string_view digest) { std::string url; - // the Location may be absolute (https://...) or registry-relative (/v2/...). + // the Location may be absolute (https://...) or registry-relative + // (/v2/...). if (location.rfind("http://", 0) == 0 || location.rfind("https://", 0) == 0) { url = std::string{location}; @@ -76,8 +77,7 @@ std::string resolve_upload_url(std::string_view registry_url, return url; } -std::optional> -parse_tags_list(std::string_view body) { +std::optional> parse_tags_list(std::string_view body) { auto j = nlohmann::json::parse(body, nullptr, /*allow_exceptions=*/false); if (j.is_discarded() || !j.is_object()) { return std::nullopt; @@ -136,7 +136,8 @@ split_registry(std::string_view configured_url) { return util::unexpected(u.error()); } - // the repository prefix is the URL path with its surrounding slashes trimmed. + // the repository prefix is the URL path with its surrounding slashes + // trimmed. std::string prefix = u->path; if (!prefix.empty() && prefix.front() == '/') { prefix.erase(prefix.begin()); @@ -152,7 +153,8 @@ split_registry(std::string_view configured_url) { if (u->port) { base += ':' + std::to_string(*u->port); } - return registry_location{.base = std::move(base), .prefix = std::move(prefix)}; + return registry_location{.base = std::move(base), + .prefix = std::move(prefix)}; } std::string repository_path(std::string_view prefix, std::string_view nspace, @@ -256,15 +258,14 @@ util::expected client::get_blob(const digest& d) { return util::unexpected{resp.error().message}; } if (resp->status != 200) { - return util::unexpected{fmt::format( - "failed to fetch blob {} (status {}): {}", d.string(), resp->status, - util::curl::http_message(resp->status))}; + return util::unexpected{ + fmt::format("failed to fetch blob {} (status {}): {}", d.string(), + resp->status, util::curl::http_message(resp->status))}; } return resp->body; } -util::expected -client::get_blob_to_file( +util::expected client::get_blob_to_file( const digest& d, const std::filesystem::path& file, std::function progress, std::function should_abort) { @@ -290,16 +291,17 @@ client::get_blob_to_file( req.on_download_progress = std::move(progress); req.should_abort = std::move(should_abort); - // hash the blob as it streams to disk so its content can be verified against - // the requested digest without a second read pass over a multi-GB file. + // hash the blob as it streams to disk so its content can be verified + // against the requested digest without a second read pass over a multi-GB + // file. const bool verify = d.algorithm() == "sha256"; util::sha256_state hash; if (verify) { util::sha256_init(hash); req.on_download_data = [&hash](const char* p, std::size_t n) { - util::sha256_update( - hash, std::span{ - reinterpret_cast(p), n}); + util::sha256_update(hash, + std::span{ + reinterpret_cast(p), n}); }; } @@ -308,9 +310,9 @@ client::get_blob_to_file( return util::unexpected{resp.error().message}; } if (resp->status != 200) { - return util::unexpected{fmt::format( - "failed to fetch blob {} (status {}): {}", d.string(), resp->status, - util::curl::http_message(resp->status))}; + return util::unexpected{ + fmt::format("failed to fetch blob {} (status {}): {}", d.string(), + resp->status, util::curl::http_message(resp->status))}; } if (verify) { @@ -374,8 +376,8 @@ util::expected, std::string> client::list_tags() { return util::unexpected{resp.error().message}; } if (resp->status != 200) { - return util::unexpected{fmt::format( - "failed to list tags (status {})", resp->status)}; + return util::unexpected{ + fmt::format("failed to list tags (status {})", resp->status)}; } auto tags = impl::parse_tags_list(resp->body); if (!tags) { @@ -480,17 +482,17 @@ client::mount_blob(const digest& d, const std::string& from_repository) { if (!resp) { return util::unexpected{resp.error().message}; } - // 201: the blob was mounted. 202: the registry declined and opened an upload - // session instead (caller must copy the blob the slow way). + // 201: the blob was mounted. 202: the registry declined and opened an + // upload session instead (caller must copy the blob the slow way). if (resp->status == 201) { return true; } if (resp->status == 202) { return false; } - return util::unexpected{fmt::format( - "failed to mount blob {} from {} (status {}): {}", d.string(), - from_repository, resp->status, resp->body)}; + return util::unexpected{ + fmt::format("failed to mount blob {} from {} (status {}): {}", + d.string(), from_repository, resp->status, resp->body)}; } util::expected @@ -507,9 +509,9 @@ client::put_blob(const digest& d, const std::filesystem::path& file) { if (!token) { return util::unexpected{token.error()}; } - return do_put_blob(registry_url_, impl::uploads_path(repository_), *token, - d.string(), - [&file](util::curl::request& r) { r.upload_file = file; }); + return do_put_blob( + registry_url_, impl::uploads_path(repository_), *token, d.string(), + [&file](util::curl::request& r) { r.upload_file = file; }); } util::expected diff --git a/src/oci/client.h b/src/oci/client.h index 2111e89f..39a91319 100644 --- a/src/oci/client.h +++ b/src/oci/client.h @@ -47,8 +47,9 @@ struct registry_location { }; // Split a configured registry URL ("host", "host/prefix", or scheme-prefixed) -// into an https base URL and a repository prefix. The scheme, if any, is dropped -// and https is always used for the base (matching how pull resolves it). +// into an https base URL and a repository prefix. The scheme, if any, is +// dropped and https is always used for the base (matching how pull resolves +// it). util::expected split_registry(std::string_view configured_url); @@ -93,8 +94,7 @@ class client { // multi-GB squashfs layer). follows the 307 redirect to backing storage. // an optional progress callback receives (bytes_downloaded, bytes_total); // an optional abort predicate, polled during transfer, cancels it. - util::expected - get_blob_to_file( + util::expected get_blob_to_file( const digest& d, const std::filesystem::path& file, std::function progress = {}, std::function should_abort = {}); @@ -125,8 +125,8 @@ class client { put_blob(const digest& d, const std::filesystem::path& file); // upload an in-memory blob (config blobs, small payloads). - util::expected - put_blob_bytes(const digest& d, const std::string& data); + util::expected put_blob_bytes(const digest& d, + const std::string& data); // PUT a manifest under `ref` (tag or digest). util::expected @@ -151,12 +151,14 @@ class client { // return a bearer token for this repository, fetching/caching as needed. // returns std::nullopt for an anonymous registry (no auth challenge), in // which case requests are sent without an Authorization header. - util::expected, std::string> token_for(bool write); + util::expected, std::string> + token_for(bool write); std::string registry_url_; // normalised, no trailing '/' std::string repository_; std::optional creds_; - // the auth challenge, or nullopt when the registry permits anonymous access. + // the auth challenge, or nullopt when the registry permits anonymous + // access. std::optional challenge_; std::optional pull_token_; std::optional push_token_; diff --git a/src/oci/digest.h b/src/oci/digest.h index 57e8c433..140bed76 100644 --- a/src/oci/digest.h +++ b/src/oci/digest.h @@ -28,10 +28,11 @@ class digest { // input that might be malformed. static digest sha256(std::string hex); - // parse ":". rejects an unknown algorithm, and a value whose - // length does not match the algorithm or that is not lowercase hex. a thin - // forwarder to oci::parse_digest. - static util::expected parse(std::string_view text); + // parse ":". rejects an unknown algorithm, and a value + // whose length does not match the algorithm or that is not lowercase hex. a + // thin forwarder to oci::parse_digest. + static util::expected + parse(std::string_view text); const std::string& algorithm() const { return algorithm_; @@ -44,7 +45,7 @@ class digest { friend bool operator==(const digest&, const digest&) = default; friend util::expected - parse_digest(std::string_view); + parse_digest(std::string_view); private: digest(std::string algorithm, std::string hex) diff --git a/src/oci/manifest.cpp b/src/oci/manifest.cpp index f89730ac..d7a6397a 100644 --- a/src/oci/manifest.cpp +++ b/src/oci/manifest.cpp @@ -68,8 +68,8 @@ nlohmann::ordered_json descriptor_json(const descriptor& d) { return j; } -nlohmann::ordered_json annotations_json( - const std::map& annotations) { +nlohmann::ordered_json +annotations_json(const std::map& annotations) { nlohmann::ordered_json j; // std::map iterates in sorted key order, matching oras. for (const auto& [k, v] : annotations) { @@ -146,8 +146,7 @@ parse_descriptor(const nlohmann::json& j) { return d; } -std::map -parse_annotations(const nlohmann::json& j) { +std::map parse_annotations(const nlohmann::json& j) { std::map out; if (auto a = j.find("annotations"); a != j.end() && a->is_object()) { for (const auto& [k, v] : a->items()) { @@ -171,12 +170,13 @@ util::expected parse_manifest(std::string_view body) { m.media_type = j.value("mediaType", std::string{media_type_manifest}); m.artifact_type = j.value("artifactType", std::string{}); - // reject multi-arch image indexes / manifest lists: they carry a `manifests` - // array of per-platform descriptors instead of `layers`, so treating one as - // an image manifest would silently yield an empty layer set. uenv only ever - // deals with single-platform artifacts. + // reject multi-arch image indexes / manifest lists: they carry a + // `manifests` array of per-platform descriptors instead of `layers`, so + // treating one as an image manifest would silently yield an empty layer + // set. uenv only ever deals with single-platform artifacts. if (m.media_type == media_type_index || - m.media_type == "application/vnd.docker.distribution.manifest.list.v2+json" || + m.media_type == + "application/vnd.docker.distribution.manifest.list.v2+json" || j.contains("manifests")) { return util::unexpected{ "multi-arch image indexes (manifest lists) are not supported"}; @@ -195,11 +195,11 @@ util::expected parse_manifest(std::string_view body) { for (const auto& l : *ls) { auto dg = digest::parse(l.value("digest", std::string{})); if (!dg) { - return util::unexpected{ - fmt::format("invalid layer digest: {}", dg.error().message())}; + return util::unexpected{fmt::format("invalid layer digest: {}", + dg.error().message())}; } - manifest_layer layer{.media_type = l.value("mediaType", - std::string{}), + manifest_layer layer{.media_type = + l.value("mediaType", std::string{}), .digest = *dg, .size = l.value("size", std::size_t{0}), .annotations = parse_annotations(l)}; diff --git a/src/oci/manifest.h b/src/oci/manifest.h index 5592c36d..1d665a42 100644 --- a/src/oci/manifest.h +++ b/src/oci/manifest.h @@ -15,8 +15,8 @@ namespace oci { // --- media types + the canonical empty config -------------------------------- -// oras-style artifact manifests carry an "empty" JSON config object. This is the -// well-known descriptor for the two-byte body "{}". +// oras-style artifact manifests carry an "empty" JSON config object. This is +// the well-known descriptor for the two-byte body "{}". inline constexpr std::string_view media_type_empty = "application/vnd.oci.empty.v1+json"; inline constexpr std::string_view empty_config_body = "{}"; @@ -89,13 +89,13 @@ struct manifest { // layers, optional subject, annotations. std::string serialize_manifest(const manifest& m); -// Parse an OCI image-manifest JSON document. Fails if the JSON is malformed or a -// descriptor digest is invalid. +// Parse an OCI image-manifest JSON document. Fails if the JSON is malformed or +// a descriptor digest is invalid. util::expected parse_manifest(std::string_view body); -// Serialize an OCI image index (application/vnd.oci.image.index.v1+json) listing -// `manifests`. Used for the referrers-tag fallback on registries that do not -// implement the Referrers API. +// Serialize an OCI image index (application/vnd.oci.image.index.v1+json) +// listing `manifests`. Used for the referrers-tag fallback on registries that +// do not implement the Referrers API. std::string serialize_index(const std::vector& manifests); } // namespace oci diff --git a/src/oci/parse.cpp b/src/oci/parse.cpp index 7ee22b19..433366c8 100644 --- a/src/oci/parse.cpp +++ b/src/oci/parse.cpp @@ -28,8 +28,8 @@ bool is_alnum_tok(lex::tok t) { return t == lex::tok::symbol || t == lex::tok::integer; } -// OCI tag grammar body: [a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}. underscore is part of -// a symbol token. +// OCI tag grammar body: [a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}. underscore is part +// of a symbol token. bool is_tag_tok(lex::tok t) { return t == lex::tok::symbol || t == lex::tok::integer || t == lex::tok::dot || t == lex::tok::dash; @@ -42,7 +42,8 @@ bool is_scheme_tok(lex::tok t) { t == lex::tok::plus || t == lex::tok::dash || t == lex::tok::dot; } -// a URL reg-name host component (unreserved + pct-encoded); we stop at ':', '/', +// a URL reg-name host component (unreserved + pct-encoded); we stop at ':', +// '/', // '?', '#', whitespace or end. bool is_regname_tok(lex::tok t) { return t == lex::tok::symbol || t == lex::tok::integer || @@ -165,21 +166,24 @@ util::expected parse_digest(std::string_view text) { const auto want = hex_length(*algorithm); if (want == 0) { return util::unexpected(parse_error{ - L.string(), fmt::format("unsupported digest algorithm '{}'", *algorithm), + L.string(), + fmt::format("unsupported digest algorithm '{}'", *algorithm), algo_tok}); } if (hex->size() != want) { - return util::unexpected(parse_error{ - L.string(), - fmt::format("expected {} hex characters, got {}", want, hex->size()), - hex_tok}); + return util::unexpected( + parse_error{L.string(), + fmt::format("expected {} hex characters, got {}", want, + hex->size()), + hex_tok}); } if (!is_lower_hex(*hex)) { return util::unexpected(parse_error{ L.string(), "the digest value is not lowercase hex", hex_tok}); } - // parse_digest is a friend of digest, so it can use the private constructor. + // parse_digest is a friend of digest, so it can use the private + // constructor. return digest{std::move(*algorithm), std::move(*hex)}; } @@ -195,8 +199,7 @@ parse_reference(std::string_view text) { const auto start = L.peek(); // the OCI tag grammar forbids a leading '.' or '-'. - if (L != lex::tok::symbol && - L != lex::tok::integer) { + if (L != lex::tok::symbol && L != lex::tok::integer) { return util::unexpected(parse_error{ L.string(), fmt::format("'{}' is not a valid tag or digest reference", s), @@ -249,8 +252,8 @@ util::expected parse_url(std::string_view text) { url u; // optional scheme: "://". consume a scheme-shaped run, and only - // accept it as a scheme if it is followed by "://"; otherwise rewind (a bare - // "host/prefix" or "host:port" begins with the same tokens). + // accept it as a scheme if it is followed by "://"; otherwise rewind (a + // bare "host/prefix" or "host:port" begins with the same tokens). { const unsigned start = L.peek().loc; std::string scheme; @@ -309,8 +312,8 @@ util::expected parse_url(std::string_view text) { } else { auto host = util::parse_string(L, "host", is_regname_tok); if (!host) { - return util::unexpected(parse_error{ - L.string(), "the url has no host", host_tok}); + return util::unexpected( + parse_error{L.string(), "the url has no host", host_tok}); } u.host = std::move(*host); } @@ -329,8 +332,8 @@ util::expected parse_url(std::string_view text) { const auto* last = digits.data() + digits.size(); if (auto [ptr, ec] = std::from_chars(first, last, port); ec != std::errc{} || ptr != last) { - return util::unexpected(parse_error{ - L.string(), "invalid port number", port_tok}); + return util::unexpected( + parse_error{L.string(), "invalid port number", port_tok}); } u.port = port; } @@ -379,8 +382,7 @@ parse_bearer_challenge(std::string_view text) { full, "expected a 'Bearer' authentication scheme", scheme_tok}); } L.next(); // consume the scheme - if (L != lex::tok::whitespace && - L != lex::tok::end) { + if (L != lex::tok::whitespace && L != lex::tok::end) { const auto t = L.peek(); return util::unexpected(parse_error{ full, "expected whitespace after the 'Bearer' scheme", t}); @@ -394,8 +396,8 @@ parse_bearer_challenge(std::string_view text) { // parameter name. const auto key_tok = L.peek(); if (L != lex::tok::symbol) { - return util::unexpected(parse_error{ - full, "expected a parameter name", key_tok}); + return util::unexpected( + parse_error{full, "expected a parameter name", key_tok}); } const std::string key = util::to_lower(L.next().spelling); @@ -404,8 +406,8 @@ parse_bearer_challenge(std::string_view text) { } if (L != lex::tok::equals) { const auto t = L.peek(); - return util::unexpected(parse_error{ - full, "expected '=' after the parameter name", t}); + return util::unexpected( + parse_error{full, "expected '=' after the parameter name", t}); } L.next(); // consume '=' if (L == lex::tok::whitespace) { @@ -470,8 +472,8 @@ parse_bearer_challenge(std::string_view text) { } } else if (L != lex::tok::end) { const auto t = L.peek(); - return util::unexpected(parse_error{ - full, "expected ',' between parameters", t}); + return util::unexpected( + parse_error{full, "expected ',' between parameters", t}); } } diff --git a/src/oci/parse.h b/src/oci/parse.h index ef64a921..1e4beb4e 100644 --- a/src/oci/parse.h +++ b/src/oci/parse.h @@ -13,25 +13,26 @@ #include // Lexer-based parsers for the string types used by the OCI client. These mirror -// the interface style of src/uenv/parse.* and share the util parsing scaffolding -// (util::parse_error, the PARSE macro, util::parse_string) so that src/oci stays -// self-contained on src/util. JSON documents are still parsed with nlohmann; the -// parsers here handle the character-level string types only. +// the interface style of src/uenv/parse.* and share the util parsing +// scaffolding (util::parse_error, the PARSE macro, util::parse_string) so that +// src/oci stays self-contained on src/util. JSON documents are still parsed +// with nlohmann; the parsers here handle the character-level string types only. namespace oci { // parse an OCI content digest ":". rejects an unknown algorithm -// and a value whose length does not match the algorithm or that is not lowercase -// hex. +// and a value whose length does not match the algorithm or that is not +// lowercase hex. util::expected parse_digest(std::string_view text); -// parse a manifest reference: a valid ":" becomes a digest reference, -// otherwise the text is validated against the OCI tag grammar and becomes a tag. +// parse a manifest reference: a valid ":" becomes a digest +// reference, otherwise the text is validated against the OCI tag grammar and +// becomes a tag. util::expected parse_reference(std::string_view text); // A URL parsed into its RFC-3986 components. `host` retains the surrounding -// brackets for an IPv6 literal (e.g. "[::1]"). Percent-encoded octets are passed -// through verbatim (not decoded). +// brackets for an IPv6 literal (e.g. "[::1]"). Percent-encoded octets are +// passed through verbatim (not decoded). struct url { std::string scheme; // e.g. "https" (may be empty) std::string userinfo; // e.g. "user:pass" (may be empty) diff --git a/src/oci/pull.cpp b/src/oci/pull.cpp index 54205aa4..9418660b 100644 --- a/src/oci/pull.cpp +++ b/src/oci/pull.cpp @@ -122,7 +122,8 @@ pull_meta(client& c, const digest& manifest_digest, const fs::path& store) { return util::unexpected{parsed.error()}; } - // the meta layer is the gzipped tar marked for unpacking (else the sole one). + // the meta layer is the gzipped tar marked for unpacking (else the sole + // one). const manifest_layer* layer = parsed->find_unpack_layer(); if (!layer && !parsed->layers.empty()) { layer = &parsed->layers[0]; diff --git a/src/oci/pull.h b/src/oci/pull.h index be2c192d..bdba1edf 100644 --- a/src/oci/pull.h +++ b/src/oci/pull.h @@ -17,9 +17,9 @@ using progress_fn = std::function; // Download the squashfs layer of `image` into /store.squashfs and // self-verify: the file is re-digested and checked against the layer digest, so -// a truncated or corrupt download is caught locally. `image` is the parsed image -// manifest. An optional abort predicate, polled during the download, cancels it -// (for Ctrl-C handling). +// a truncated or corrupt download is caught locally. `image` is the parsed +// image manifest. An optional abort predicate, polled during the download, +// cancels it (for Ctrl-C handling). util::expected pull_squashfs(client& c, const manifest& image, const std::filesystem::path& store, progress_fn progress = {}, @@ -28,8 +28,8 @@ pull_squashfs(client& c, const manifest& image, // Download and unpack the `uenv/meta` referrer (a gzipped tar) into , // reproducing the `meta/` directory. Returns true if meta was found and // pulled, false if the image has no attached meta. -util::expected -pull_meta(client& c, const digest& manifest_digest, - const std::filesystem::path& store); +util::expected pull_meta(client& c, + const digest& manifest_digest, + const std::filesystem::path& store); } // namespace oci diff --git a/src/oci/push.cpp b/src/oci/push.cpp index 1dd28877..572093e2 100644 --- a/src/oci/push.cpp +++ b/src/oci/push.cpp @@ -60,8 +60,8 @@ util::expected put_empty_config(client& c) { std::string{empty_config_body}); } -// A packaged payload ready to become a manifest layer: the blob on disk plus the -// layer descriptor, and a scratch directory to clean up after upload. +// A packaged payload ready to become a manifest layer: the blob on disk plus +// the layer descriptor, and a scratch directory to clean up after upload. struct packaged_layer { fs::path blob; manifest_layer layer; @@ -88,8 +88,8 @@ package_directory(const fs::path& dir) { "--owner=0", "--group=0", "--numeric-owner", "-cf", tar_path.string(), "-C", parent.string(), name}); if (!tar) { - return util::unexpected{fmt::format( - "unable to run tar to pack {}: {}", dir.string(), tar.error())}; + return util::unexpected{fmt::format("unable to run tar to pack {}: {}", + dir.string(), tar.error())}; } if (auto rc = tar->wait(); rc != 0) { return util::unexpected{ @@ -105,7 +105,8 @@ package_directory(const fs::path& dir) { // gzip in place; -n omits the filename/timestamp for a reproducible result. auto gz = util::run({"gzip", "-n", tar_path.string()}); if (!gz) { - return util::unexpected{fmt::format("unable to run gzip: {}", gz.error())}; + return util::unexpected{ + fmt::format("unable to run gzip: {}", gz.error())}; } if (auto rc = gz->wait(); rc != 0) { return util::unexpected{fmt::format("gzip failed (exit {})", rc)}; @@ -125,14 +126,15 @@ package_directory(const fs::path& dir) { return packaged_layer{ .blob = gz_path, - .layer = manifest_layer{.media_type = std::string{media_type_layer_targz}, - .digest = *layer_digest, - .size = size, - .annotations = - {{std::string{annotation_content_digest}, - tar_digest->string()}, - {std::string{annotation_unpack}, "true"}, - {std::string{annotation_title}, name}}}, + .layer = + manifest_layer{ + .media_type = std::string{media_type_layer_targz}, + .digest = *layer_digest, + .size = size, + .annotations = {{std::string{annotation_content_digest}, + tar_digest->string()}, + {std::string{annotation_unpack}, "true"}, + {std::string{annotation_title}, name}}}, .scratch = *work}; } @@ -146,8 +148,8 @@ util::expected package_file(const fs::path& file) { std::error_code ec; auto size = fs::file_size(file, ec); if (ec) { - return util::unexpected{fmt::format("unable to stat {}: {}", - file.string(), ec.message())}; + return util::unexpected{ + fmt::format("unable to stat {}: {}", file.string(), ec.message())}; } std::string media_type{media_type_octet_stream}; @@ -157,12 +159,11 @@ util::expected package_file(const fs::path& file) { return packaged_layer{ .blob = file, - .layer = manifest_layer{ - .media_type = media_type, - .digest = *layer_digest, - .size = size, - .annotations = {{std::string{annotation_title}, - file.filename().string()}}}}; + .layer = manifest_layer{.media_type = media_type, + .digest = *layer_digest, + .size = size, + .annotations = {{std::string{annotation_title}, + file.filename().string()}}}}; } // the blob digests a manifest references (config + layers). @@ -178,9 +179,9 @@ std::vector blob_digests(const manifest& m) { // copy one blob from `from_repo` into `dst`'s repository, preferring a // server-side cross-repo mount and falling back to streaming it through a local // temp file when the registry declines the mount. -util::expected -copy_blob(client& src, client& dst, const std::string& from_repo, - const digest& d) { +util::expected copy_blob(client& src, client& dst, + const std::string& from_repo, + const digest& d) { auto mounted = dst.mount_blob(d, from_repo); if (!mounted) { return util::unexpected{mounted.error()}; @@ -282,9 +283,10 @@ push_squashfs(client& c, const fs::path& squashfs, const reference& ref) { return digest_of_string(body); } -util::expected -attach(client& c, const reference& subject, std::string_view artifact_type, - const fs::path& payload) { +util::expected attach(client& c, + const reference& subject, + std::string_view artifact_type, + const fs::path& payload) { // stat the payload once, and branch file-vs-directory off that single // result (avoids a TOCTOU between exists() and is_directory()). std::error_code ec; @@ -306,8 +308,9 @@ attach(client& c, const reference& subject, std::string_view artifact_type, subject.string(), subj.error())}; } descriptor subject_desc{ - .media_type = subj->media_type.empty() ? std::string{media_type_manifest} - : subj->media_type, + .media_type = subj->media_type.empty() + ? std::string{media_type_manifest} + : subj->media_type, .digest = subj->digest ? *subj->digest : digest_of_string(subj->body), .size = subj->body.size()}; @@ -378,8 +381,9 @@ copy_image(const std::string& registry_base, const std::string& src_repo, // fetch the image manifest and move its blobs. auto mr = src->get_manifest(reference::digest(src_manifest)); if (!mr) { - return util::unexpected{fmt::format("unable to fetch source manifest {}: {}", - src_manifest.string(), mr.error())}; + return util::unexpected{ + fmt::format("unable to fetch source manifest {}: {}", + src_manifest.string(), mr.error())}; } const digest manifest_digest = mr->digest.value_or(src_manifest); const std::string_view manifest_media = @@ -395,8 +399,8 @@ copy_image(const std::string& registry_base, const std::string& src_repo, return util::unexpected{ok.error()}; } } - // PUT the image manifest under the destination tag. Content is byte-for-byte - // identical, so the digest (identity) is preserved. + // PUT the image manifest under the destination tag. Content is + // byte-for-byte identical, so the digest (identity) is preserved. if (auto ok = dst->put_manifest(reference::tag(dst_tag), mr->body, manifest_media); !ok) { @@ -406,7 +410,8 @@ copy_image(const std::string& registry_base, const std::string& src_repo, // copy referrers (the --recursive part): meta and any other attachments. auto refs = src->referrers(manifest_digest); if (!refs) { - // a registry without the Referrers API just has nothing to recurse into. + // a registry without the Referrers API just has nothing to recurse + // into. spdlog::debug("copy_image: no referrers for {} ({})", manifest_digest.string(), refs.error()); return {}; @@ -414,9 +419,9 @@ copy_image(const std::string& registry_base, const std::string& src_repo, for (const auto& r : *refs) { auto rm = src->get_manifest(reference::digest(r.digest)); if (!rm) { - return util::unexpected{fmt::format( - "unable to fetch referrer {}: {}", r.digest.string(), - rm.error())}; + return util::unexpected{ + fmt::format("unable to fetch referrer {}: {}", + r.digest.string(), rm.error())}; } auto rparsed = parse_manifest(rm->body); if (!rparsed) { @@ -427,9 +432,9 @@ copy_image(const std::string& registry_base, const std::string& src_repo, return util::unexpected{ok.error()}; } } - const std::string_view rmedia = - rm->media_type.empty() ? media_type_manifest - : std::string_view{rm->media_type}; + const std::string_view rmedia = rm->media_type.empty() + ? media_type_manifest + : std::string_view{rm->media_type}; // the referrer's subject digest is the image manifest digest, which is // unchanged by the copy, so the manifest is valid verbatim in dst. if (auto ok = dst->put_manifest(reference::digest(r.digest), rm->body, diff --git a/src/oci/reference.h b/src/oci/reference.h index 4c0caaf2..3fe16766 100644 --- a/src/oci/reference.h +++ b/src/oci/reference.h @@ -22,8 +22,8 @@ class reference { // build a reference from a content digest. static reference digest(oci::digest d); // parse text: a valid ":" becomes a digest reference, otherwise - // it is validated against the OCI tag grammar and becomes a tag reference. a - // thin forwarder to oci::parse_reference. + // it is validated against the OCI tag grammar and becomes a tag reference. + // a thin forwarder to oci::parse_reference. static util::expected parse(std::string_view text); diff --git a/src/uenv/parse.cpp b/src/uenv/parse.cpp index 00c09148..80cef693 100644 --- a/src/uenv/parse.cpp +++ b/src/uenv/parse.cpp @@ -241,8 +241,7 @@ parse_uenv_nslabel(const std::string& in) { ++i; } // check for following colons - if (L.peek(i) == lex::tok::colon && - L.peek(i + 1) == lex::tok::colon) { + if (L.peek(i) == lex::tok::colon && L.peek(i + 1) == lex::tok::colon) { // parse the namespace name PARSE(L, name, nspace); // gobble the :: diff --git a/src/uenv/settings.cpp b/src/uenv/settings.cpp index 83c68a77..129bd960 100644 --- a/src/uenv/settings.cpp +++ b/src/uenv/settings.cpp @@ -601,8 +601,9 @@ parse_registry(const toml::node& input) { if (auto v = value.value()) { listing_url = v.value(); } else { - return make_config_error("registry.listing_url must be a string", - value.source().begin.line); + return make_config_error( + "registry.listing_url must be a string", + value.source().begin.line); } } else if (key == "default_namespace") { if (auto v = value.value()) { diff --git a/src/util/curl.cpp b/src/util/curl.cpp index 06072461..e9f1ab2c 100644 --- a/src/util/curl.cpp +++ b/src/util/curl.cpp @@ -264,8 +264,8 @@ size_t file_write_callback(void* source, size_t size, size_t n, void* target) { return written; } -int xferinfo_callback(void* p, curl_off_t dltotal, curl_off_t dlnow, - curl_off_t, curl_off_t) { +int xferinfo_callback(void* p, curl_off_t dltotal, curl_off_t dlnow, curl_off_t, + curl_off_t) { const auto& req = *static_cast(p); if (req.on_download_progress) { req.on_download_progress(static_cast(dlnow), @@ -304,7 +304,8 @@ expected perform(const request& req) { CURL* h = curl_easy_init(); if (!h) { - return unexpected{error{CURLE_FAILED_INIT, "unable to initialise curl"}}; + return unexpected{ + error{CURLE_FAILED_INIT, "unable to initialise curl"}}; } auto cleanup_handle = defer([h]() { curl_easy_cleanup(h); }); @@ -321,8 +322,8 @@ expected perform(const request& req) { CURL_EASY(curl_easy_setopt(h, CURLOPT_NOBODY, 1L)); break; default: - CURL_EASY( - curl_easy_setopt(h, CURLOPT_CUSTOMREQUEST, method_name(req.method))); + CURL_EASY(curl_easy_setopt(h, CURLOPT_CUSTOMREQUEST, + method_name(req.method))); break; } @@ -333,8 +334,7 @@ expected perform(const request& req) { if (req.follow_redirects) { CURL_EASY(curl_easy_setopt(h, CURLOPT_FOLLOWLOCATION, 1L)); CURL_EASY(curl_easy_setopt(h, CURLOPT_MAXREDIRS, 10L)); - CURL_EASY( - curl_easy_setopt(h, CURLOPT_REDIR_PROTOCOLS_STR, "https")); + CURL_EASY(curl_easy_setopt(h, CURLOPT_REDIR_PROTOCOLS_STR, "https")); } // assemble request headers @@ -349,8 +349,8 @@ expected perform(const request& req) { } if (req.bearer_token) { slist = curl_slist_append( - slist, fmt::format("Authorization: Bearer {}", *req.bearer_token) - .c_str()); + slist, + fmt::format("Authorization: Bearer {}", *req.bearer_token).c_str()); } if (slist) { CURL_EASY(curl_easy_setopt(h, CURLOPT_HTTPHEADER, slist)); @@ -374,10 +374,9 @@ expected perform(const request& req) { if (req.upload_file) { upload = fopen(req.upload_file->c_str(), "rb"); if (!upload) { - return unexpected{ - error{CURLE_READ_ERROR, - fmt::format("unable to open {} for upload", - req.upload_file->string())}}; + return unexpected{error{CURLE_READ_ERROR, + fmt::format("unable to open {} for upload", + req.upload_file->string())}}; } curl_off_t file_size = std::filesystem::file_size(*req.upload_file); CURL_EASY(curl_easy_setopt(h, CURLOPT_UPLOAD, 1L)); @@ -385,8 +384,8 @@ expected perform(const request& req) { CURL_EASY(curl_easy_setopt(h, CURLOPT_INFILESIZE_LARGE, file_size)); } else if (req.body) { CURL_EASY(curl_easy_setopt(h, CURLOPT_POSTFIELDS, req.body->data())); - CURL_EASY(curl_easy_setopt(h, CURLOPT_POSTFIELDSIZE, - (long)req.body->size())); + CURL_EASY( + curl_easy_setopt(h, CURLOPT_POSTFIELDSIZE, (long)req.body->size())); } // capture the response body: either streamed to a file (for large blob @@ -402,16 +401,16 @@ expected perform(const request& req) { if (req.download_file) { download = fopen(req.download_file->c_str(), "wb"); if (!download) { - return unexpected{ - error{CURLE_WRITE_ERROR, - fmt::format("unable to open {} for download", - req.download_file->string())}}; + return unexpected{error{ + CURLE_WRITE_ERROR, fmt::format("unable to open {} for download", + req.download_file->string())}}; } download_sink.file = download; download_sink.on_data = &req.on_download_data; CURL_EASY( curl_easy_setopt(h, CURLOPT_WRITEFUNCTION, file_write_callback)); - CURL_EASY(curl_easy_setopt(h, CURLOPT_WRITEDATA, (void*)&download_sink)); + CURL_EASY( + curl_easy_setopt(h, CURLOPT_WRITEDATA, (void*)&download_sink)); } else { body.reserve(200000); CURL_EASY(curl_easy_setopt(h, CURLOPT_WRITEFUNCTION, memory_callback)); @@ -423,8 +422,8 @@ expected perform(const request& req) { CURL_EASY(curl_easy_setopt(h, CURLOPT_HEADERFUNCTION, header_callback)); CURL_EASY(curl_easy_setopt(h, CURLOPT_HEADERDATA, (void*)&resp.headers)); - CURL_EASY(curl_easy_setopt(h, CURLOPT_CONNECTTIMEOUT_MS, - req.connect_timeout_ms)); + CURL_EASY( + curl_easy_setopt(h, CURLOPT_CONNECTTIMEOUT_MS, req.connect_timeout_ms)); CURL_EASY(curl_easy_setopt(h, CURLOPT_TIMEOUT_MS, req.timeout_ms)); if (req.on_download_progress || req.should_abort) { diff --git a/src/util/curl.h b/src/util/curl.h index ec05c8e6..3763134d 100644 --- a/src/util/curl.h +++ b/src/util/curl.h @@ -27,13 +27,14 @@ struct error { // client (the token dance, blob HEAD/PUT, manifest PUT/GET, referrers/tags). // Unlike the convenience helpers below, perform() does NOT treat HTTP >= 400 as // an error: a 401 auth challenge or a 404 blob-miss are normal steps in the OCI -// flow, so the status and response headers are always handed back to the caller. +// flow, so the status and response headers are always handed back to the +// caller. enum class http_method { get, head, post, put, patch, del }; // Case-insensitive view of HTTP response headers. Header names are stored -// lower-cased (HTTP field names are case-insensitive); the last value seen for a -// given name wins. +// lower-cased (HTTP field names are case-insensitive); the last value seen for +// a given name wins. struct headers { std::unordered_map entries; @@ -53,7 +54,8 @@ struct request { std::optional username; std::optional password; - // request body. at most one of these should be set (used by put/post/patch): + // request body. at most one of these should be set (used by + // put/post/patch): // - body: an in-memory payload (e.g. a manifest blob) // - upload_file: stream a file as the body (e.g. a squashfs layer) std::optional body; diff --git a/src/util/fs.cpp b/src/util/fs.cpp index cff70ddb..55f5a5e0 100644 --- a/src/util/fs.cpp +++ b/src/util/fs.cpp @@ -86,13 +86,13 @@ ensure_directory(const std::filesystem::path& path) { std::error_code ec; fs::create_directories(path, ec); if (ec) { - return unexpected( - fmt::format("unable to create {}: {}", path.string(), ec.message())); + return unexpected(fmt::format("unable to create {}: {}", path.string(), + ec.message())); } if (!fs::is_directory(path, ec)) { - return unexpected(fmt::format( - "{} exists but is not a directory", path.string())); + return unexpected( + fmt::format("{} exists but is not a directory", path.string())); } if (file_access_level(path) != file_level::readwrite) { @@ -115,8 +115,7 @@ unsquashfs_tmp(const std::filesystem::path& sqfs, auto base = make_temp_dir(); if (!base) { - return unexpected( - fmt::format("unsquashfs_tmp: {}", base.error())); + return unexpected(fmt::format("unsquashfs_tmp: {}", base.error())); } std::vector command{ "unsquashfs", "-no-exit", "-d", base->string(), diff --git a/src/util/sha256.cpp b/src/util/sha256.cpp index 8cb87f90..dbadb731 100644 --- a/src/util/sha256.cpp +++ b/src/util/sha256.cpp @@ -82,7 +82,8 @@ void sha256_impl::process_block(const uint8_t block[64]) { for (int i = 16; i < 64; ++i) { uint32_t s0 = rotr(w[i - 15], 7) ^ rotr(w[i - 15], 18) ^ (w[i - 15] >> 3); - uint32_t s1 = rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ (w[i - 2] >> 10); + uint32_t s1 = + rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ (w[i - 2] >> 10); w[i] = w[i - 16] + s0 + w[i - 7] + s1; } diff --git a/test/unit/curl.cpp b/test/unit/curl.cpp index 899ce265..cc971d4c 100644 --- a/test/unit/curl.cpp +++ b/test/unit/curl.cpp @@ -21,11 +21,12 @@ TEST_CASE("curl parse_header_line", "[curl]") { // the OCI-relevant headers the registry client needs to capture util::curl::parse_header_line( - h, "WWW-Authenticate: Bearer realm=\"https://r/token\",service=\"r\"\r\n"); + h, + "WWW-Authenticate: Bearer realm=\"https://r/token\",service=\"r\"\r\n"); util::curl::parse_header_line( h, "Location: https://r/v2/x/blobs/uploads/abc123\r\n"); - util::curl::parse_header_line( - h, "Docker-Content-Digest: sha256:deadbeef\r\n"); + util::curl::parse_header_line(h, + "Docker-Content-Digest: sha256:deadbeef\r\n"); REQUIRE(h.get("www-authenticate") == "Bearer realm=\"https://r/token\",service=\"r\""); @@ -43,7 +44,8 @@ TEST_CASE("curl parse_header_line edge cases", "[curl]") { // surrounding whitespace is trimmed from name and value; a value may itself // contain a colon (e.g. a URL or digest). - util::curl::parse_header_line(h, " Location : https://host:443/path \r\n"); + util::curl::parse_header_line(h, + " Location : https://host:443/path \r\n"); REQUIRE(h.get("location") == "https://host:443/path"); // empty value is preserved diff --git a/test/unit/oci_auth.cpp b/test/unit/oci_auth.cpp index 56a39942..8cc4bb4e 100644 --- a/test/unit/oci_auth.cpp +++ b/test/unit/oci_auth.cpp @@ -22,10 +22,9 @@ using oci::impl::repository_scope; using oci::impl::token_url; TEST_CASE("oci token_url", "[oci][auth]") { - oci::bearer_challenge c{ - .realm = "https://jfrog.svc.cscs.ch/v2/token", - .service = "jfrog.svc.cscs.ch", - .scopes = {}}; + oci::bearer_challenge c{.realm = "https://jfrog.svc.cscs.ch/v2/token", + .service = "jfrog.svc.cscs.ch", + .scopes = {}}; // matches the spike's proven-working token request URL REQUIRE(token_url(c, {"repository:uenv/deploy/x/24.7:pull"}) == @@ -74,8 +73,9 @@ TEST_CASE("oci get_credentials reads a token file", "[oci][auth]") { // only the first line is used as the token write_file(token_file, "s3cr3t-token\nsecond line ignored\n"); - auto c = oci::get_credentials(std::optional{"alice"}, - std::optional{token_file.string()}); + auto c = + oci::get_credentials(std::optional{"alice"}, + std::optional{token_file.string()}); REQUIRE(c); // no error REQUIRE(c->has_value()); // credentials resolved REQUIRE((*c)->username == "alice"); @@ -83,8 +83,8 @@ TEST_CASE("oci get_credentials reads a token file", "[oci][auth]") { } TEST_CASE("oci get_credentials with no token is anonymous", "[oci][auth]") { - // no --token -> nullopt credentials (anonymous), never an error, even when a - // username is supplied. + // no --token -> nullopt credentials (anonymous), never an error, even when + // a username is supplied. auto anon = oci::get_credentials(std::nullopt, std::nullopt); REQUIRE(anon); REQUIRE_FALSE(anon->has_value()); @@ -95,11 +95,10 @@ TEST_CASE("oci get_credentials with no token is anonymous", "[oci][auth]") { REQUIRE_FALSE(anon_user->has_value()); } -TEST_CASE("oci get_credentials errors on a missing token path", - "[oci][auth]") { - auto c = - oci::get_credentials(std::optional{"alice"}, - std::optional{"/no/such/path-xyz123"}); +TEST_CASE("oci get_credentials errors on a missing token path", "[oci][auth]") { + auto c = oci::get_credentials( + std::optional{"alice"}, + std::optional{"/no/such/path-xyz123"}); REQUIRE_FALSE(c); // a --token that is not a path/file is an error } @@ -128,11 +127,11 @@ TEST_CASE("oci get_credentials falls back to the login name", "[oci][auth]") { auto token_file = dir / "tok"; write_file(token_file, "tok\n"); - auto c = oci::get_credentials(std::nullopt, - std::optional{token_file.string()}); + auto c = oci::get_credentials( + std::nullopt, std::optional{token_file.string()}); // getlogin() resolves a username in a normal session, but may be empty in a - // detached environment (e.g. some CI) -> the function then reports an error. - // Assert deterministically on both outcomes. + // detached environment (e.g. some CI) -> the function then reports an + // error. Assert deterministically on both outcomes. if (c) { REQUIRE(c->has_value()); REQUIRE_FALSE((*c)->username.empty()); @@ -147,7 +146,7 @@ TEST_CASE("oci credentials formatter redacts the password", "[oci][auth]") { auto s = fmt::format("{}", c); REQUIRE(s.find("alice") != std::string::npos); REQUIRE(s.find("secret") == std::string::npos); // password must not leak - REQUIRE(s.find("XXXXXX") != std::string::npos); // 6 chars -> 6 'X' + REQUIRE(s.find("XXXXXX") != std::string::npos); // 6 chars -> 6 'X' } TEST_CASE("oci resolve_credentials precedence", "[oci][auth]") { @@ -188,7 +187,8 @@ TEST_CASE("oci resolve_credentials precedence", "[oci][auth]") { auto dir = util::make_temp_dir().value(); auto cfg = dir / "config.json"; // echo -n carol:pw | base64 -> Y2Fyb2w6cHc= - write_file(cfg, R"({"auths":{"reg.example.com":{"auth":"Y2Fyb2w6cHc="}}})"); + write_file(cfg, + R"({"auths":{"reg.example.com":{"auth":"Y2Fyb2w6cHc="}}})"); oci::credential_sources src; src.docker_config = cfg; auto c = oci::resolve_credentials(host, src); @@ -202,8 +202,9 @@ TEST_CASE("oci resolve_credentials precedence", "[oci][auth]") { SECTION("docker config host normalisation") { auto dir = util::make_temp_dir().value(); auto cfg = dir / "config.json"; - write_file(cfg, - R"({"auths":{"https://reg.example.com/v2/":{"auth":"Y2Fyb2w6cHc="}}})"); + write_file( + cfg, + R"({"auths":{"https://reg.example.com/v2/":{"auth":"Y2Fyb2w6cHc="}}})"); oci::credential_sources src; src.docker_config = cfg; auto c = oci::resolve_credentials(host, src); @@ -227,7 +228,8 @@ TEST_CASE("oci resolve_credentials precedence", "[oci][auth]") { SECTION("docker config host miss") { auto dir = util::make_temp_dir().value(); auto cfg = dir / "config.json"; - write_file(cfg, R"({"auths":{"other.example.com":{"auth":"Y2Fyb2w6cHc="}}})"); + write_file( + cfg, R"({"auths":{"other.example.com":{"auth":"Y2Fyb2w6cHc="}}})"); oci::credential_sources src; src.docker_config = cfg; auto c = oci::resolve_credentials(host, src); @@ -239,8 +241,8 @@ TEST_CASE("oci resolve_credentials precedence", "[oci][auth]") { SECTION("docker config credsStore entry skipped") { auto dir = util::make_temp_dir().value(); auto cfg = dir / "config.json"; - write_file(cfg, - R"({"auths":{"reg.example.com":{}},"credsStore":"desktop"})"); + write_file( + cfg, R"({"auths":{"reg.example.com":{}},"credsStore":"desktop"})"); oci::credential_sources src; src.docker_config = cfg; auto c = oci::resolve_credentials(host, src); diff --git a/test/unit/oci_client.cpp b/test/unit/oci_client.cpp index bfd77442..6d36c291 100644 --- a/test/unit/oci_client.cpp +++ b/test/unit/oci_client.cpp @@ -46,13 +46,15 @@ TEST_CASE("oci resolve_upload_url", "[oci][client]") { "deadbeef"); // relative Location that already carries a query -> '&' - REQUIRE(resolve_upload_url(reg, "/v2/r/blobs/uploads/abc?_state=xyz", dig) == - "https://jfrog.svc.cscs.ch/v2/r/blobs/uploads/abc?_state=xyz&digest=" - "sha256:deadbeef"); + REQUIRE( + resolve_upload_url(reg, "/v2/r/blobs/uploads/abc?_state=xyz", dig) == + "https://jfrog.svc.cscs.ch/v2/r/blobs/uploads/abc?_state=xyz&digest=" + "sha256:deadbeef"); // absolute Location (e.g. redirected to storage) is used verbatim - REQUIRE(resolve_upload_url(reg, "https://store.example/up/abc?sig=1", dig) == - "https://store.example/up/abc?sig=1&digest=sha256:deadbeef"); + REQUIRE( + resolve_upload_url(reg, "https://store.example/up/abc?sig=1", dig) == + "https://store.example/up/abc?sig=1&digest=sha256:deadbeef"); // trailing slash on the registry base is not doubled REQUIRE(resolve_upload_url("https://reg/", "/v2/x", dig) == @@ -60,8 +62,7 @@ TEST_CASE("oci resolve_upload_url", "[oci][client]") { } TEST_CASE("oci parse_tags_list", "[oci][client]") { - auto tags = - parse_tags_list(R"({"name":"r","tags":["v1","v2","latest"]})"); + auto tags = parse_tags_list(R"({"name":"r","tags":["v1","v2","latest"]})"); REQUIRE(tags.has_value()); REQUIRE(*tags == std::vector{"v1", "v2", "latest"}); diff --git a/test/unit/oci_manifest.cpp b/test/unit/oci_manifest.cpp index f6b355a0..6a380ce7 100644 --- a/test/unit/oci_manifest.cpp +++ b/test/unit/oci_manifest.cpp @@ -52,12 +52,12 @@ TEST_CASE("serialize meta referrer manifest is oras-identical", .digest = digest::sha256( "3e2e44102d0c54bc2b72295b470b994f128a89b1436d567d850dbf131fcc02db"), .size = 2366, - .annotations = { - {std::string{annotation_content_digest}, - "sha256:" - "b751bb420ec887b245ea91483849775a8904975aab23b999473b9ea07df9571f"}, - {std::string{annotation_unpack}, "true"}, - {std::string{annotation_title}, "meta"}}}); + .annotations = {{std::string{annotation_content_digest}, + "sha256:" + "b751bb420ec887b245ea91483849775a8904975aab23b999473b9" + "ea07df9571f"}, + {std::string{annotation_unpack}, "true"}, + {std::string{annotation_title}, "meta"}}}); m.subject = descriptor{.media_type = std::string{media_type_manifest}, .digest = digest::sha256(image_manifest_hex), .size = 588}; @@ -121,8 +121,7 @@ TEST_CASE("parse_manifest finds the unpack layer", "[oci][manifest]") { TEST_CASE("parse_manifest rejects malformed", "[oci][manifest]") { REQUIRE_FALSE(parse_manifest("not json").has_value()); // a layer with an invalid digest is a hard error - REQUIRE_FALSE(parse_manifest( - R"({"layers":[{"digest":"sha256:short"}]})") + REQUIRE_FALSE(parse_manifest(R"({"layers":[{"digest":"sha256:short"}]})") .has_value()); } diff --git a/test/unit/oci_parse.cpp b/test/unit/oci_parse.cpp index ce9b0077..954e32e4 100644 --- a/test/unit/oci_parse.cpp +++ b/test/unit/oci_parse.cpp @@ -125,7 +125,7 @@ TEST_CASE("oci parse_url components", "[oci][parse]") { TEST_CASE("oci parse_url rejects malformed", "[oci][parse]") { REQUIRE_FALSE(parse_url("").has_value()); - REQUIRE_FALSE(parse_url("/uenv").has_value()); // no host + REQUIRE_FALSE(parse_url("/uenv").has_value()); // no host REQUIRE_FALSE(parse_url("bad host/uenv").has_value()); // whitespace in host REQUIRE_FALSE(parse_url("host:notaport/x").has_value()); } @@ -159,8 +159,8 @@ TEST_CASE("oci parse_bearer_challenge space-separated scopes", "[oci][parse]") { "Bearer realm=\"https://r/token\",service=\"reg\"," "scope=\"repository:a:pull repository:b:pull\""); REQUIRE(c.has_value()); - REQUIRE(c->scopes == std::vector{"repository:a:pull", - "repository:b:pull"}); + REQUIRE(c->scopes == + std::vector{"repository:a:pull", "repository:b:pull"}); } TEST_CASE("oci parse_bearer_challenge rejects non-bearer / no realm", diff --git a/test/unit/oci_registry.cpp b/test/unit/oci_registry.cpp index c5892d81..4c88a33d 100644 --- a/test/unit/oci_registry.cpp +++ b/test/unit/oci_registry.cpp @@ -86,12 +86,11 @@ struct zot_instance { const auto cfg = state / "config.json"; { std::ofstream f{cfg}; - f << fmt::format( - R"({{"storage":{{"rootDirectory":"{}"}},)" - R"("http":{{"address":"127.0.0.1","port":"{}"}},)" - R"("log":{{"level":"error","output":"{}"}}}})", - (state / "registry").string(), port, - (state / "zot.log").string()); + f << fmt::format(R"({{"storage":{{"rootDirectory":"{}"}},)" + R"("http":{{"address":"127.0.0.1","port":"{}"}},)" + R"("log":{{"level":"error","output":"{}"}}}})", + (state / "registry").string(), port, + (state / "zot.log").string()); } auto p = util::run({zot, "serve", cfg.string()}); if (!p) { @@ -158,7 +157,8 @@ TEST_CASE("oci registry push/pull round-trip", "[registry]") { // the pushed digest is the sha256 of the manifest the registry stores. auto resp = c->get_manifest(oci::reference::tag("v1")); REQUIRE(resp.has_value()); - const auto local = oci::digest::from_sha256(util::sha256_string(resp->body)); + const auto local = + oci::digest::from_sha256(util::sha256_string(resp->body)); REQUIRE(local == *pushed); if (resp->digest) { REQUIRE(*resp->digest == *pushed); @@ -192,7 +192,8 @@ TEST_CASE("oci registry attach + referrers + pull_meta", "[registry]") { auto pushed = oci::push_squashfs(*c, sqfs, oci::reference::tag("v1")); REQUIRE(pushed.has_value()); - auto att = oci::attach(*c, oci::reference::digest(*pushed), "uenv/meta", meta); + auto att = + oci::attach(*c, oci::reference::digest(*pushed), "uenv/meta", meta); REQUIRE(att.has_value()); auto refs = c->referrers(*pushed); @@ -231,8 +232,8 @@ TEST_CASE("oci registry copy preserves digest", "[registry]") { auto pushed = oci::push_squashfs(*src, sqfs, oci::reference::tag("v1")); REQUIRE(pushed.has_value()); - auto copied = oci::copy_image(base, src_repo, dst_repo, *pushed, "v1", - std::nullopt); + auto copied = + oci::copy_image(base, src_repo, dst_repo, *pushed, "v1", std::nullopt); REQUIRE(copied.has_value()); // the destination manifest is byte-identical, so its digest is preserved. @@ -240,6 +241,7 @@ TEST_CASE("oci registry copy preserves digest", "[registry]") { REQUIRE(dst.has_value()); auto resp = dst->get_manifest(oci::reference::tag("v1")); REQUIRE(resp.has_value()); - const auto local = oci::digest::from_sha256(util::sha256_string(resp->body)); + const auto local = + oci::digest::from_sha256(util::sha256_string(resp->body)); REQUIRE(local == *pushed); } diff --git a/test/unit/sha256.cpp b/test/unit/sha256.cpp index 9281a325..4396497c 100644 --- a/test/unit/sha256.cpp +++ b/test/unit/sha256.cpp @@ -20,11 +20,10 @@ TEST_CASE("sha256 known vectors", "[sha256]") { "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); REQUIRE(util::sha256_string("abc").hex() == "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); - REQUIRE( - util::sha256_string( - "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq") - .hex() == - "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1"); + REQUIRE(util::sha256_string( + "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq") + .hex() == + "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1"); } TEST_CASE("sha256 zero-byte input", "[sha256]") { @@ -88,7 +87,8 @@ TEST_CASE("sha256 update splits do not change the digest", "[sha256]") { for (std::size_t split = 0; split <= msg.size(); ++split) { util::sha256_state s; util::sha256_init(s); - util::sha256_update(s, as_bytes(std::string_view{msg}.substr(0, split))); + util::sha256_update(s, + as_bytes(std::string_view{msg}.substr(0, split))); util::sha256_update(s, as_bytes(std::string_view{msg}.substr(split))); REQUIRE(util::sha256_final(s).hex() == whole); } diff --git a/test/unit/strings.cpp b/test/unit/strings.cpp index d281d954..d86c39a4 100644 --- a/test/unit/strings.cpp +++ b/test/unit/strings.cpp @@ -57,7 +57,8 @@ TEST_CASE("base64_decode", "[strings]") { REQUIRE(dec("Zm9vYmFy") == std::optional{"foobar"}); // a docker-style user:pass token - REQUIRE(dec("YWxpY2U6czNjcmV0") == std::optional{"alice:s3cret"}); + REQUIRE(dec("YWxpY2U6czNjcmV0") == + std::optional{"alice:s3cret"}); // padding is optional; embedded whitespace/newlines are ignored REQUIRE(dec("Zm9v\n") == std::optional{"foo"}); From a9c58295015d4fc4e7442e49ea636a51855d4aad Mon Sep 17 00:00:00 2001 From: bcumming Date: Sun, 5 Jul 2026 22:42:28 +0200 Subject: [PATCH 17/29] add progress bar for uploads --- src/cli/push.cpp | 45 ++++++++++++++++++++++++++++++++++++--------- src/oci/client.cpp | 13 +++++++++---- src/oci/client.h | 6 ++++-- src/oci/push.cpp | 6 ++++-- src/oci/push.h | 4 +++- src/util/curl.cpp | 11 ++++++++--- src/util/curl.h | 4 ++++ 7 files changed, 68 insertions(+), 21 deletions(-) diff --git a/src/cli/push.cpp b/src/cli/push.cpp index a68e9ad8..3ced7d05 100644 --- a/src/cli/push.cpp +++ b/src/cli/push.cpp @@ -1,5 +1,7 @@ // vim: ts=4 sts=4 sw=4 et +#include +#include #include #include @@ -18,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -155,15 +158,39 @@ int image_push([[maybe_unused]] const image_push_args& args, // interrupt leaves nothing referencing a partial blob. std::optional image_digest; { - auto spinner = bk::Animation({ - .message = fmt::format("pushing {} to registry", - sqfs->sqfs.filename().string()), - .style = bk::Ellipsis, - .no_tty = !isatty(fileno(stdout)), - }); - auto push_result = oci::push_squashfs(*client, sqfs->sqfs, - oci::reference::tag(*L.tag)); - spinner->done(); + // atomic: written by curl's progress callback on the main thread, read + // concurrently by barkeep's display thread. + std::atomic uploaded_mb{0u}; + std::error_code ec; + auto size_byte = std::filesystem::file_size(sqfs->sqfs, ec); + // round up so total_mb is never zero + std::size_t total_mb{ + ec ? 0u : (size_byte + (1024 * 1024 - 1)) / (1024 * 1024)}; + auto bar = bk::ProgressBar( + &uploaded_mb, + { + .total = total_mb, + .message = fmt::format("pushing {} to registry", + sqfs->sqfs.filename().string()), + .speed = 0.1, + .speed_unit = "MB/s", + .style = color::use_color() ? bk::ProgressBarStyle::Rich + : bk::ProgressBarStyle::Bars, + .no_tty = !isatty(fileno(stdout)), + }); + + auto progress = [&uploaded_mb](std::uint64_t now, std::uint64_t) { + uploaded_mb = now / (1024 * 1024); + }; + auto push_result = oci::push_squashfs( + *client, sqfs->sqfs, oci::reference::tag(*L.tag), progress); + if (push_result) { + // ensure the progress bar shows 100% on completion (also covers the + // case where the registry already had the blob and no upload + // happened). + uploaded_mb = total_mb; + } + bar->done(); if (!push_result) { term::error("unable to push uenv.\n{}", push_result.error()); return 1; diff --git a/src/oci/client.cpp b/src/oci/client.cpp index 4c5a71d3..ef679159 100644 --- a/src/oci/client.cpp +++ b/src/oci/client.cpp @@ -496,7 +496,8 @@ client::mount_blob(const digest& d, const std::string& from_repository) { } util::expected -client::put_blob(const digest& d, const std::filesystem::path& file) { +client::put_blob(const digest& d, const std::filesystem::path& file, + std::function progress) { if (util::file_access_level(file) < util::file_level::readonly) { return util::unexpected{fmt::format( "cannot upload blob: {} is not a readable file", file.string())}; @@ -509,9 +510,13 @@ client::put_blob(const digest& d, const std::filesystem::path& file) { if (!token) { return util::unexpected{token.error()}; } - return do_put_blob( - registry_url_, impl::uploads_path(repository_), *token, d.string(), - [&file](util::curl::request& r) { r.upload_file = file; }); + return do_put_blob(registry_url_, impl::uploads_path(repository_), *token, + d.string(), [&file, &progress](util::curl::request& r) { + r.upload_file = file; + if (progress) { + r.on_upload_progress = progress; + } + }); } util::expected diff --git a/src/oci/client.h b/src/oci/client.h index 39a91319..ac6fdd24 100644 --- a/src/oci/client.h +++ b/src/oci/client.h @@ -120,9 +120,11 @@ class client { // upload a blob via the monolithic POST-then-PUT handshake. the file is // streamed from disk (not held in memory), so large squashfs layers are - // fine. no-op if the blob already exists. + // fine. no-op if the blob already exists. an optional progress callback + // receives (bytes_uploaded, bytes_total) during the PUT. util::expected - put_blob(const digest& d, const std::filesystem::path& file); + put_blob(const digest& d, const std::filesystem::path& file, + std::function progress = {}); // upload an in-memory blob (config blobs, small payloads). util::expected put_blob_bytes(const digest& d, diff --git a/src/oci/push.cpp b/src/oci/push.cpp index 572093e2..1222b3cd 100644 --- a/src/oci/push.cpp +++ b/src/oci/push.cpp @@ -242,7 +242,8 @@ void maintain_referrers_tag(client& c, const digest& subject, } // namespace util::expected -push_squashfs(client& c, const fs::path& squashfs, const reference& ref) { +push_squashfs(client& c, const fs::path& squashfs, const reference& ref, + std::function progress) { auto layer_digest = digest_of_file(squashfs); if (!layer_digest) { return util::unexpected{layer_digest.error()}; @@ -258,7 +259,8 @@ push_squashfs(client& c, const fs::path& squashfs, const reference& ref) { layer_digest->string(), size, ref.string()); // upload the squashfs blob (streamed) and the empty config. - if (auto ok = c.put_blob(*layer_digest, squashfs); !ok) { + if (auto ok = c.put_blob(*layer_digest, squashfs, std::move(progress)); + !ok) { return util::unexpected{ok.error()}; } if (auto ok = put_empty_config(c); !ok) { diff --git a/src/oci/push.h b/src/oci/push.h index 80104e12..6ab6bb0d 100644 --- a/src/oci/push.h +++ b/src/oci/push.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -20,7 +21,8 @@ namespace oci { // manifest digest — the canonical image id. util::expected push_squashfs(client& c, const std::filesystem::path& squashfs, - const reference& ref); + const reference& ref, + std::function progress = {}); // Attach typed side-data to an existing image as an OCI referrer (the native // replacement for `oras attach`). `subject` identifies the target image (tag or diff --git a/src/util/curl.cpp b/src/util/curl.cpp index e9f1ab2c..ce1d8175 100644 --- a/src/util/curl.cpp +++ b/src/util/curl.cpp @@ -264,13 +264,17 @@ size_t file_write_callback(void* source, size_t size, size_t n, void* target) { return written; } -int xferinfo_callback(void* p, curl_off_t dltotal, curl_off_t dlnow, curl_off_t, - curl_off_t) { +int xferinfo_callback(void* p, curl_off_t dltotal, curl_off_t dlnow, + curl_off_t ultotal, curl_off_t ulnow) { const auto& req = *static_cast(p); if (req.on_download_progress) { req.on_download_progress(static_cast(dlnow), static_cast(dltotal)); } + if (req.on_upload_progress) { + req.on_upload_progress(static_cast(ulnow), + static_cast(ultotal)); + } if (req.should_abort && req.should_abort()) { return 1; // non-zero aborts the transfer (CURLE_ABORTED_BY_CALLBACK) } @@ -426,7 +430,8 @@ expected perform(const request& req) { curl_easy_setopt(h, CURLOPT_CONNECTTIMEOUT_MS, req.connect_timeout_ms)); CURL_EASY(curl_easy_setopt(h, CURLOPT_TIMEOUT_MS, req.timeout_ms)); - if (req.on_download_progress || req.should_abort) { + if (req.on_download_progress || req.on_upload_progress || + req.should_abort) { CURL_EASY(curl_easy_setopt(h, CURLOPT_NOPROGRESS, 0L)); CURL_EASY( curl_easy_setopt(h, CURLOPT_XFERINFOFUNCTION, xferinfo_callback)); diff --git a/src/util/curl.h b/src/util/curl.h index 3763134d..2feb8828 100644 --- a/src/util/curl.h +++ b/src/util/curl.h @@ -70,6 +70,10 @@ struct request { // bytes_total is 0 until the server reports a content length. std::function on_download_progress; + // optional upload-progress callback: (bytes_uploaded, bytes_total). + // driven from the same xferinfo callback; used for the upload_file path. + std::function on_upload_progress; + // optional tap on the response body as it streams to download_file: called // with each chunk actually written, before returning to libcurl. lets a // caller hash the blob on the fly and so verify its digest without a second From 03de6937e9c9372fd5e41cba0ffa3608a6115370 Mon Sep 17 00:00:00 2001 From: Ben Cumming Date: Sun, 5 Jul 2026 22:58:58 +0200 Subject: [PATCH 18/29] Update settings.h --- src/uenv/settings.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/uenv/settings.h b/src/uenv/settings.h index c0a3eed9..219dc266 100644 --- a/src/uenv/settings.h +++ b/src/uenv/settings.h @@ -15,9 +15,6 @@ struct registry_config { std::string url; std::string default_namespace; std::optional artifactory_url; - // overrides the uenv listing service base URL (default: - // https://uenv-list.svc.cscs.ch/list); used to point at a local/mock - // listing endpoint for testing. std::optional listing_url; }; From 64e8e417d929ba378accc2b759cc466d6589705f Mon Sep 17 00:00:00 2001 From: bcumming Date: Sun, 5 Jul 2026 23:03:09 +0200 Subject: [PATCH 19/29] remove wrapdb --- .gitignore | 1 + subprojects/wrapdb.json | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 subprojects/wrapdb.json diff --git a/.gitignore b/.gitignore index 59a4357f..c95f829e 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ subprojects/nlohmann_json-* subprojects/tomlplusplus-* subprojects/zlib-* subprojects/.wraplock +subprojects/.wrapdb # logs and artifacts generated by packaging workflows packaging/*.log diff --git a/subprojects/wrapdb.json b/subprojects/wrapdb.json deleted file mode 100644 index 8de73a48..00000000 --- a/subprojects/wrapdb.json +++ /dev/null @@ -1 +0,0 @@ -{"abseil-cpp":{"dependency_names":["absl_base","absl_container","absl_crc","absl_debugging","absl_flags","absl_hash","absl_log","absl_numeric","absl_profiling","absl_random","absl_status","absl_strings","absl_synchronization","absl_time","absl_types","absl_algorithm_container","absl_any_invocable","absl_bad_any_cast_impl","absl_bad_optional_access","absl_bad_variant_access","absl_bind_front","absl_city","absl_civil_time","absl_cleanup","absl_cord","absl_cord_internal","absl_cordz_functions","absl_cordz_handle","absl_cordz_info","absl_cordz_sample_token","absl_core_headers","absl_crc32c","absl_debugging_internal","absl_demangle_internal","absl_die_if_null","absl_examine_stack","absl_exponential_biased","absl_failure_signal_handler","absl_flags_commandlineflag","absl_flags_commandlineflag_internal","absl_flags_config","absl_flags_internal","absl_flags_marshalling","absl_flags_parse","absl_flags_private_handle_accessor","absl_flags_program_name","absl_flags_reflection","absl_flags_usage","absl_flags_usage_internal","absl_flat_hash_map","absl_flat_hash_set","absl_function_ref","absl_graphcycles_internal","absl_hashtablez_sampler","absl_inlined_vector","absl_int128","absl_leak_check","absl_log_initialize","absl_log_internal_check_op","absl_log_internal_message","absl_log_severity","absl_low_level_hash","absl_memory","absl_optional","absl_periodic_sampler","absl_random_bit_gen_ref","absl_random_distributions","absl_random_internal_distribution_test_util","absl_random_internal_platform","absl_random_internal_pool_urbg","absl_random_internal_randen","absl_random_internal_randen_hwaes","absl_random_internal_randen_hwaes_impl","absl_random_internal_randen_slow","absl_random_internal_seed_material","absl_random_random","absl_random_seed_gen_exception","absl_random_seed_sequences","absl_raw_hash_set","absl_raw_logging_internal","absl_scoped_set_env","absl_span","absl_spinlock_wait","absl_stacktrace","absl_statusor","absl_str_format","absl_str_format_internal","absl_strerror","absl_string_view","absl_strings_internal","absl_symbolize","absl_throw_delegate","absl_time_zone","absl_type_traits","absl_utility","absl_variant"],"versions":["20250814.1-1","20250127.1-2","20250127.1-1","20240722.0-4","20240722.0-3","20240722.0-2","20240722.0-1","20230802.1-2","20230802.1-1","20230802.0-3","20230802.0-2","20230802.0-1","20230125.1-5","20230125.1-4","20230125.1-3","20230125.1-2","20230125.1-1","20220623.0-2","20220623.0-1","20211102.0-3","20211102.0-2","20211102.0-1","20210324.2-4","20210324.2-3","20210324.2-2","20210324.2-1","20210324.1-4","20210324.1-3","20210324.1-2","20210324.1-1","20200923.2-1","20200225.2-3","20200225.2-2","20200225.2-1"]},"adamyaxley-obfuscate":{"dependency_names":["Obfuscate"],"versions":["1.0.0-1"]},"aklomp-base64":{"dependency_names":["base64"],"versions":["0.5.2-1"]},"apache-orc":{"dependency_names":["orc"],"versions":["2.3.0-1","2.2.2-1","2.2.1-1","2.2.0-1"]},"arduinocore-avr":{"dependency_names":["arduinocore","arduinocore-main","arduinocore-eeprom","arduinocore-hid","arduinocore-softwareserial","arduinocore-spi","arduinocore-wire"],"versions":["1.8.7-1","1.8.6-1","1.8.2-2","1.8.2-1","1.6.20-1"]},"argparse":{"dependency_names":["argparse"],"versions":["3.2-2","3.2-1","3.1-2","3.1-1","3.0-1","2.9-1","2.6-1","2.5-1","2.4-1","2.2-1"]},"argtable3":{"dependency_names":["argtable3"],"versions":["3.3.1-1","3.3.0-1","3.2.2-1"]},"argu-parser":{"dependency_names":["argu-parser"],"versions":["0.0.1-1"]},"argus":{"dependency_names":["argus"],"versions":["0.2.1-1","0.1.0-1"]},"arrow-cpp":{"dependency_names":["arrow","arrow-compute"],"versions":["21.0.0-1"]},"asio":{"dependency_names":["asio"],"versions":["1.36.0-2","1.36.0-1","1.30.2-2","1.30.2-1","1.28.1-1","1.24.0-1"]},"atomic_queue":{"dependency_names":["atomic_queue"],"versions":["1.9.2-1","1.7.1-1","1.7.0-1","1.6.9-1","1.6.7-1","1.6.6-1"]},"au":{"dependency_names":["au"],"versions":["0.5.0-1","0.4.1-1"]},"aws-c-cal":{"dependency_names":["aws-c-cal"],"versions":["0.9.10-1"]},"aws-c-common":{"dependency_names":["aws-c-common"],"versions":["0.12.4-3","0.12.4-2","0.12.4-1"]},"aws-c-compression":{"dependency_names":["aws-c-compression"],"program_names":["aws-c-common-huffman-generator"],"versions":["0.3.1-1"]},"backward-cpp":{"dependency_names":["backward-cpp","backward-cpp-interface"],"versions":["1.6-6","1.6-5","1.6-4","1.6-3","1.6-2","1.6-1"]},"bdwgc":{"dependency_names":["bdw-gc"],"versions":["8.2.10-1","8.2.8-1","8.2.2-1","8.2.0-1","7.6.8-1"]},"blueprint-compiler":{"program_names":["blueprint-compiler"],"versions":["0.18.0-2","0.18.0-1","0.16.0-1","0.10.0-1"]},"boost-mp11":{"dependency_names":["boost-mp11"],"versions":["1.88.0-1","1.83.0-1"]},"boost-pfr":{"dependency_names":["boost-pfr"],"versions":["1.88.0-1"]},"box2d":{"dependency_names":["box2d"],"versions":["2.4.1-4","2.4.1-3","2.4.1-2","2.4.1-1","2.3.1-7","2.3.1-5","2.3.1-4","2.3.1-2"]},"bshoshany-thread-pool":{"dependency_names":["bshoshany-thread-pool"],"versions":["5.1.0-1","5.0.0-1","4.1.0-1","4.0.1-1","4.0.0-1","3.5.0-1","3.4.0-1","3.3.0-1"]},"bzip2":{"dependency_names":["bzip2"],"versions":["1.0.8-2","1.0.8-1"]},"c-ares":{"dependency_names":["libcares"],"versions":["1.34.6-1","1.34.5-1","1.34.4-1","1.24.0-1","1.22.1-2","1.22.1-1","1.20.1-1"]},"c-flags":{"dependency_names":["c-flags"],"versions":["1.5.10-1","1.5.8-1","1.5.4-1"]},"cairo":{"dependency_names":["cairo","cairo-gobject","cairo-script-interpreter"],"versions":["1.18.4-2","1.18.4-1","1.18.2-1","1.18.0-1","1.17.8-1"]},"cairomm":{"dependency_names":["cairomm-1.16"],"versions":["1.18.0-1"]},"catch":{"versions":["2.2.2-1","2.2.1-2","2.2.1-1"]},"catch2":{"dependency_names":["catch2","catch2-with-main"],"versions":["3.14.0-1","3.13.0-1","3.12.0-1","3.11.0-1","3.10.0-1","3.9.1-1","3.8.1-1","3.8.0-1","3.7.1-1","3.7.0-1","3.6.0-1","3.5.4-1","3.5.3-1","3.5.2-1","3.4.0-1","3.3.2-1","3.2.0-1","3.1.0-1","2.13.8-1","2.13.7-1","2.13.3-2","2.13.3-1","2.11.3-1","2.11.1-1","2.9.0-1","2.8.0-1","2.7.2-1","2.5.0-1","2.4.1-1"]},"catchorg-clara":{"versions":["1.1.5-1"]},"centurion":{"dependency_names":["centurion"],"versions":["7.3.0-1","7.2.0-1"]},"cereal":{"dependency_names":["cereal"],"versions":["1.3.2-1","1.3.0-1","1.2.2-1"]},"cexception":{"dependency_names":["cexception"],"versions":["1.3.4-1","1.3.3-1"]},"cglm":{"dependency_names":["cglm"],"versions":["0.9.6-1","0.9.4-1","0.9.2-1"]},"check":{"dependency_names":["check"],"versions":["0.15.2-6","0.15.2-5","0.15.2-4","0.15.2-3","0.15.2-2","0.15.2-1","0.14.0-1"]},"chipmunk":{"dependency_names":["chipmunk"],"versions":["7.0.3-1","6.2.2-2","6.2.2-1"]},"cjson":{"dependency_names":["libcjson","libcjson_utils"],"versions":["1.7.19-1","1.7.18-2","1.7.18-1","1.7.17-1","1.7.16-2","1.7.16-1","1.7.15-6","1.7.15-5","1.7.15-4","1.7.15-3","1.7.15-2","1.7.15-1","1.7.14-1"]},"cli11":{"dependency_names":["CLI11"],"versions":["2.6.2-1","2.6.1-1","2.6.0-1","2.5.0-2","2.5.0-1","2.4.2-1","2.4.1-1","2.3.2-1","2.2.0-1","2.1.2-1","1.9.1-1"]},"cmark-gfm":{"dependency_names":["libcmark-gfm"],"versions":["0.29.0.gfm.13-1","0.29.0.gfm.10-1","0.29.0.gfm.6-1"]},"cmock":{"dependency_names":["cmock"],"versions":["2.6.0-1","2.5.3-1"]},"cmocka":{"dependency_names":["cmocka"],"versions":["1.1.8-1","1.1.7-6","1.1.7-5","1.1.7-4","1.1.7-3","1.1.7-2","1.1.7-1","1.1.5-5","1.1.5-4","1.1.5-3","1.1.5-2","1.1.5-1","1.1.2-1"]},"concurrentqueue":{"dependency_names":["concurrentqueue"],"versions":["1.0.4-1","1.0.3-1"]},"cpp-httplib":{"dependency_names":["cpp-httplib"],"versions":["0.48.0-1","0.47.0-1","0.46.1-1","0.46.0-1","0.45.0-1","0.43.2-1","0.43.1-1","0.42.0-1","0.41.0-1","0.40.0-1","0.39.0-1","0.38.0-1","0.37.2-1","0.37.1-1","0.37.0-1","0.36.0-1","0.35.0-1","0.34.0-1","0.33.1-1","0.32.0-1","0.31.0-1","0.30.2-1","0.30.1-1","0.30.0-1","0.29.0-1","0.28.0-1","0.27.0-1","0.26.0-1","0.25.0-1","0.24.0-1","0.23.1-1","0.22.0-1","0.21.0-1","0.20.1-1","0.20.0-1","0.19.0-1","0.18.7-1","0.18.6-1","0.18.5-1","0.18.3-1","0.18.2-1","0.18.1-1","0.18.0-1","0.17.3-1","0.17.1-1","0.16.3-1","0.15.3-1","0.13.1-1","0.11.2-1","0.8.9-2","0.8.9-1"]},"cpp-semver":{"dependency_names":["cpp-semver"],"versions":["0.3.3-1"]},"cpputest":{"dependency_names":["cpputest"],"versions":["4.0-4","4.0-3","4.0-2","4.0-1"]},"cppzmq":{"dependency_names":["cppzmq"],"versions":["4.11.0-2","4.11.0-1","4.10.0-4","4.10.0-3","4.10.0-2","4.10.0-1","4.9.0-1","4.8.1-2","4.8.1-1"]},"cpr":{"dependency_names":["cpr"],"versions":["1.14.2-1","1.14.1-1","1.12.0-1","1.11.2-1","1.11.1-1","1.11.0-1","1.10.5-1","1.10.4-1","1.9.6-2","1.9.6-1","1.9.3-1","1.9.2-2","1.9.2-1","1.7.2-1","1.5.0-1","1.3.0-1"]},"croaring":{"dependency_names":["croaring"],"versions":["4.3.11-1","1.3.0-2","1.3.0-1"]},"crow":{"dependency_names":["Crow"],"versions":["1.3.0-1"]},"csfml":{"dependency_names":["csfml"],"versions":["3.0.0-1"]},"ctre":{"dependency_names":["ctre"],"versions":["3.10.0-1","3.9.0-1"]},"curl":{"dependency_names":["libcurl"],"versions":["8.12.1-2","8.12.1-1","8.10.1-1","8.10.0-1","8.9.1-2","8.9.1-1","8.9.0-1","8.8.0-1","8.7.1-1","8.6.0-1","8.5.0-2","8.5.0-1","8.4.0-2","8.4.0-1","8.3.0-1","8.0.1-1"]},"cwalk":{"dependency_names":["cwalk"],"versions":["1.2.9-2","1.2.9-1"]},"cxxopts":{"dependency_names":["cxxopts"],"versions":["3.2.0-1","3.1.1-1","3.0.0-1","2.2.1-1","2.2.0-2","2.2.0-1"]},"debug_assert":{"dependency_names":["debug_assert"],"versions":["1.3.4-1","1.3.3-1"]},"directxmath":{"dependency_names":["directxmath"],"versions":["3.1.9-2","3.1.9-1","3.1.8-1"]},"dlfcn-win32":{"dependency_names":["dl"],"versions":["1.4.2-1","1.4.1-1","1.3.1-1"]},"docopt":{"dependency_names":["docopt"],"versions":["0.6.3-5","0.6.3-4","0.6.3-3","0.6.3-2","0.6.3-1"]},"doctest":{"dependency_names":["doctest"],"versions":["2.5.2-1","2.5.1-1","2.5.0-1","2.4.12-1","2.4.11-1","2.4.9-1","2.4.8-1"]},"dragonbox":{"dependency_names":["dragonbox"],"versions":["1.1.2-1"]},"dyninst":{"dependency_names":["dyninst"],"versions":["13.0.0-2","13.0.0-1"]},"eigen":{"dependency_names":["eigen3"],"versions":["5.0.1-1","5.0.0-1","3.4.0-2","3.4.0-1","3.3.9-1","3.3.8-1","3.3.7-3","3.3.7-2","3.3.7-1","3.3.5-1","3.3.4-1"]},"emilk-loguru":{"dependency_names":["loguru"],"versions":["2.1.0-10","2.1.0-9","2.1.0-8","2.1.0-7","2.1.0-6","2.1.0-5","2.1.0-4","2.1.0-3","2.1.0-2","2.1.0-1"]},"enet":{"dependency_names":["enet"],"versions":["1.3.18-1","1.3.17-2","1.3.17-1","1.3.13-4","1.3.13-2"]},"enlog":{"dependency_names":["enlog"],"versions":["1.8-1","1.6-1"]},"entt":{"dependency_names":["entt"],"versions":["3.11.0-3","3.11.0-2","3.11.0-1"]},"epoxy":{"dependency_names":["epoxy"],"versions":["1.5.10-2","1.5.10-1","1.5.9-1"]},"exiv2":{"dependency_names":["exiv2"],"versions":["0.28.8-1","0.28.7-1","0.28.5-1","0.28.4-1","0.28.3-1","0.28.1-1","0.28.0-1"]},"expat":{"dependency_names":["expat"],"versions":["2.8.2-1","2.7.5-1","2.7.4-1","2.7.3-1","2.7.1-1","2.6.4-1","2.6.3-2","2.6.3-1","2.6.2-1","2.6.0-1","2.5.0-4","2.5.0-3","2.5.0-2","2.5.0-1","2.4.9-1","2.4.8-2","2.4.8-1","2.2.9-4","2.2.9-3","2.2.9-2","2.2.9-1","2.2.6-1","2.2.5-5","2.2.5-4","2.2.5-3","2.2.5-1"]},"eyalroz-printf":{"dependency_names":["eyalroz-printf"],"versions":["6.2.0-1"]},"facil":{"dependency_names":["facil"],"versions":["0.7.6-5","0.7.6-4","0.7.6-3","0.7.6-2","0.7.6-1","0.7.5-1"]},"fast_float":{"dependency_names":["FastFloat"],"versions":["8.2.4-1"]},"fdk-aac":{"dependency_names":["fdk-aac"],"versions":["2.0.3-2","2.0.3-1","2.0.2-0"]},"ff-nvcodec-headers":{"dependency_names":["ffnvcodec"],"versions":["11.1.5.1-0"]},"flac":{"dependency_names":["flac"],"versions":["1.5.0-3","1.5.0-2","1.5.0-1","1.4.3-2","1.4.3-1","1.4.2-2","1.4.2-1","1.4.1-1","1.4.0-2","1.4.0-1","1.3.4-4","1.3.4-3","1.3.4-2","1.3.4-1","1.3.3-2","1.3.3-1"]},"flatbuffers":{"dependency_names":["flatbuffers"],"program_names":["flatc","flathash"],"versions":["24.3.25-1","24.3.6-1","23.3.3-1","23.1.21-1","22.11.23-1","2.0.8-1","2.0.6-2","2.0.6-1","1.11.0-1"]},"fluidsynth":{"dependency_names":["fluidsynth"],"versions":["2.3.3-2","2.3.3-1","2.3.2-1","2.3.0-1","2.2.8-1","2.2.6-1","2.2.4-3","2.2.4-2","2.2.4-1","2.2.3-2","2.2.3-1","2.2.0-1","2.1.8-1","2.1.7-1"]},"fmt":{"dependency_names":["fmt"],"versions":["12.0.0-1","11.2.0-2","11.2.0-1","11.1.4-1","11.1.1-2","11.1.1-1","11.0.2-1","11.0.1-1","10.2.0-2","10.2.0-1","10.1.1-1","9.1.0-2","9.1.0-1","9.0.0-1","8.1.1-2","8.1.1-1","8.0.1-1","7.1.3-1","7.0.1-1","6.2.0-2","6.2.0-1","6.0.0-1","5.3.0-1","5.2.1-1","5.2.0-1","4.1.0-1"]},"fontconfig":{"dependency_names":["fontconfig"],"versions":["2.16.2-1","2.16.1-1","2.16.0-1","2.15.0-1","2.14.2-1"]},"freeglut":{"dependency_names":["freeglut","glut"],"versions":["3.8.0-2","3.8.0-1","3.6.0-1","3.4.0-3","3.4.0-2","3.4.0-1"]},"freetype2":{"dependency_names":["freetype2"],"versions":["2.14.3-1","2.14.2-1","2.14.1-1","2.14.0-1","2.13.3-2","2.13.3-1","2.13.2-1","2.13.1-1","2.13.0-1","2.12.1-2","2.12.1-1","2.12.0-1","2.11.1-1","2.11.0-1","2.9.1-2","2.9.1-1"]},"fribidi":{"dependency_names":["fribidi"],"versions":["1.0.16-1","1.0.15-1","1.0.13-1"]},"frozen":{"dependency_names":["frozen"],"versions":["1.2.0-1","1.1.1-1","1.0.1-2","1.0.1-1"]},"ftxui":{"dependency_names":["ftxui-screen","ftxui-dom","ftxui-component"],"versions":["6.1.9-1","5.0.0-1","4.0.0-1","3.0.0-2","3.0.0-1","2.0.0-3","2.0.0-2","2.0.0-1"]},"fuse":{"dependency_names":["fuse"],"versions":["2.9.9-4","2.9.9-3","2.9.9-2","2.9.9-1"]},"gdbm":{"dependency_names":["gdbm"],"versions":["1.26-2","1.26-1","1.24-1","1.23-2","1.23-1","1.14.1-2","1.14.1-1"]},"gdk-pixbuf":{"dependency_names":["gdk-pixbuf-2.0"],"versions":["2.44.7-1","2.44.6-1","2.44.5-1","2.44.4-1","2.44.3-1","2.44.2-1","2.44.1-1","2.44.0-1","2.42.12-1","2.42.10-1","2.42.9-1"]},"gee":{"dependency_names":["gee-0.8"],"versions":["0.20.8-1","0.20.6-1"]},"gflags":{"dependency_names":["gflags"],"versions":["2.2.2-1"]},"giflib":{"dependency_names":["giflib"],"versions":["5.2.2-3","5.2.2-2","5.2.2-1","5.2.1-3","5.2.1-2","5.2.1-1"]},"glbinding":{"dependency_names":["glbinding","glbinding-aux"],"versions":["3.5.0-1","3.3.0-1"]},"glew":{"dependency_names":["glew"],"versions":["2.2.0-2","2.2.0-1","2.1.0-4","2.1.0-3","2.1.0-2","2.1.0-1"]},"glfw":{"dependency_names":["glfw3"],"versions":["3.4-2","3.4-1","3.3.10-1","3.3.9-1","3.3.8-3","3.3.8-2","3.3.8-1","3.3.7-1"]},"glib":{"dependency_names":["gthread-2.0","gobject-2.0","gmodule-no-export-2.0","gmodule-export-2.0","gmodule-2.0","glib-2.0","gio-2.0","gio-windows-2.0","gio-unix-2.0"],"program_names":["glib-genmarshal","glib-mkenums","glib-compile-schemas","glib-compile-resources","gio-querymodules","gdbus-codegen"],"versions":["2.88.2-1","2.88.1-1","2.88.0-1","2.86.4-1","2.86.3-1","2.86.2-1","2.86.1-1","2.86.0-1","2.84.4-1","2.84.3-1","2.84.2-1","2.84.1-1","2.84.0-1","2.82.5-1","2.82.4-1","2.82.2-1","2.82.1-1","2.82.0-1","2.80.5-1","2.80.4-1","2.80.3-1","2.80.2-1","2.80.0-1","2.78.4-1","2.78.3-1","2.78.2-1","2.78.1-1","2.78.0-1","2.76.5-1","2.76.4-1","2.76.3-1","2.76.1-1","2.74.4-1","2.74.1-1","2.74.0-2","2.74.0-1","2.72.2-1","2.72.1-1","2.70.4-1","2.70.2-1","2.70.1-1","2.70.0-1","2.68.1-1","2.66.7-1"]},"glib-networking":{"versions":["2.80.1-1","2.80.0-1","2.78.0-1"]},"glm":{"dependency_names":["glm"],"versions":["1.0.1-1","1.0.0-1","0.9.9.8-2","0.9.9.8-1"]},"glpk":{"dependency_names":["glpk"],"versions":["5.0-2","5.0-1"]},"gobject-introspection":{"dependency_names":["gobject-introspection-1.0"],"program_names":["g-ir-annotation-tool","g-ir-compiler","g-ir-generate","g-ir-inspect","g-ir-scanner","g-ir-doc-tool"],"versions":["1.86.0-1"]},"godot-cpp":{"dependency_names":["godot-cpp"],"versions":["4.5-1","4.4.1-2","4.4.1-1","4.3-1","4.2-1"]},"google-benchmark":{"dependency_names":["benchmark","benchmark_main"],"versions":["1.8.4-5","1.8.4-4","1.8.4-3","1.8.4-2","1.8.4-1","1.8.3-1","1.7.1-2","1.7.1-1","1.7.0-1","1.6.0-1","1.5.2-1","1.4.1-1"]},"google-brotli":{"dependency_names":["libbrotlicommon","libbrotlienc","libbrotlidec"],"program_names":["brotli"],"versions":["1.1.0-3","1.1.0-2","1.1.0-1","1.0.9-2","1.0.9-1","1.0.7-1"]},"google-snappy":{"dependency_names":["snappy"],"versions":["1.2.2-2","1.2.2-1","1.1.9-1","1.1.7-2","1.1.7-1"]},"google-woff2":{"dependency_names":["libwoff2common","libwoff2dec","libwoff2enc"],"program_names":["woff2_decompress","woff2_compress","woff2_info"],"versions":["1.0.2-4","1.0.2-3","1.0.2-2","1.0.2-1"]},"graphite2":{"dependency_names":["graphite2"],"versions":["1.3.14-1"]},"grpc":{"dependency_names":["grpc","grpc_unsecure","grpc++","grpc++_unsecure"],"program_names":["grpc_cpp_plugin","grpc_node_plugin","grpc_php_plugin","grpc_python_plugin","grpc_ruby_plugin"],"versions":["1.59.1-1"]},"gtest":{"dependency_names":["gtest","gtest_main","gmock","gmock_main"],"versions":["1.17.0-4","1.17.0-3","1.17.0-2","1.17.0-1","1.15.2-4","1.15.2-3","1.15.2-2","1.15.2-1","1.15.0-1","1.14.0-2","1.14.0-1","1.13.0-1","1.12.1-1","1.11.0-2","1.11.0-1","1.10.0-1","1.8.1-1","1.8.0-5","1.8.0-4","1.8.0-3","1.8.0-2","1.8.0-1","1.7.0-5","1.7.0-4","1.7.0-2"]},"gumbo-parser":{"dependency_names":["gumbo"],"versions":["0.12.1-2","0.12.1-1","0.10.1-1"]},"harfbuzz":{"dependency_names":["harfbuzz","harfbuzz-cairo","harfbuzz-gobject","harfbuzz-icu","harfbuzz-subset"],"versions":["13.0.1-1","12.3.2-1","12.3.1-1","12.3.0-1","12.2.0-1","12.1.0-1","11.5.0-1","11.4.5-1","11.4.4-1","11.4.3-1","11.4.1-1","11.3.3-1","11.3.2-1","11.2.1-1","11.2.0-1","11.1.0-1","11.0.1-1","11.0.0-1","10.4.0-1","10.3.0-1","10.2.0-1","10.1.0-1","10.0.1-1","9.0.0-1","8.3.1-1","8.3.0-1","8.2.2-1","8.2.1-1","8.2.0-1","8.1.1-1","5.2.0-1","4.4.1-1"]},"hedley":{"dependency_names":["hedley"],"versions":["15-1","11-1"]},"hinnant-date":{"dependency_names":["date","tz"],"versions":["3.0.3-1","3.0.1-2","3.0.1-1","3.0.0-1","2.4.1-1"]},"htslib":{"dependency_names":["htslib"],"versions":["1.22.1-1","1.22-1","1.21-2","1.21-1","1.20-1","1.17-2","1.17-1","1.16-3","1.16-2","1.16-1","1.15-1","1.14-1","1.11-1","1.10.2-1","1.9-1"]},"icu":{"dependency_names":["icu-data","icu-i18n","icu-io","icu-uc"],"program_names":["genbrk","genccode","gencmn"],"versions":["78.2-1","78.1-1","77.1-3","77.1-2","77.1-1","76.1-2","76.1-1","73.2-2","73.2-1","73.1-1","72.1-5","72.1-4","72.1-3","72.1-2","72.1-1","71.1-1","70.1-2","70.1-1","67.1-4","67.1-3","67.1-2","67.1-1","55.2-1"]},"iir":{"dependency_names":["iir"],"versions":["1.9.3-2","1.9.3-1","1.9.2-2","1.9.2-1"]},"imgui":{"dependency_names":["imgui"],"versions":["1.92.5-1","1.91.6-3","1.91.6-2","1.91.6-1","1.91.3-1","1.91.0-1","1.89.9-2","1.89.9-1","1.89.3-1","1.89.2-1","1.88-2","1.88-1","1.87-5","1.87-4","1.87-3","1.87-2","1.87-1","1.86-1","1.85-1","1.81-1","1.80-1","1.79-2","1.79-1","1.78-2","1.78-1","1.76-2","1.76-1"]},"imgui-docking":{"dependency_names":["imgui-docking"],"versions":["1.92.5-1","1.92.3-3","1.92.3-2","1.92.3-1","1.91.6-1","1.91.0-1"]},"imgui-sfml":{"dependency_names":["imgui-sfml"],"versions":["3.0-1","2.6.1-1","2.6-2","2.6-1","2.5-4","2.5-3","2.5-2","2.5-1","2.3-2","2.3-1","2.1-1"]},"imguizmo":{"dependency_names":["imguizmo"],"versions":["1.83-1"]},"implot":{"dependency_names":["implot"],"versions":["0.17-1","0.16-1"]},"indicators":{"dependency_names":["indicators"],"versions":["2.3-1","2.2-2","2.2-1"]},"inih":{"dependency_names":["inih","inireader"],"versions":["r62-1","r61-1","r59-1","r58-1","r57-1","r56-1","r54-1","r53-1","r52-1","r51-1"]},"irepeat":{"dependency_names":["irepeat"],"versions":["0.6-1"]},"jansson":{"dependency_names":["jansson"],"versions":["2.14.1-1","2.14-3","2.14-2","2.14-1","2.13-1","2.11-3","2.11-2","2.11-1"]},"jbig2dec":{"dependency_names":["jbig2dec"],"versions":["0.20-1"]},"jbigkit":{"dependency_names":["libjbig","libjbig85"],"program_names":["jbgtopbm","pbmtojbg","jbgtopbm85","pbmtojbg85"],"versions":["2.1-2","2.1-1"]},"jitterentropy":{"dependency_names":["jitterentropy"],"versions":["3.6.3-2","3.6.3-1"]},"json":{"versions":["3.2.0-1","2.1.1-1","2.0.5-1","2.0.3-1"]},"json-c":{"dependency_names":["json-c"],"versions":["0.18-2","0.18-1","0.17-2","0.17-1","0.16-4","0.16-3","0.16-2","0.16-1","0.15-2","0.15-1","0.13.1-1"]},"json-glib":{"dependency_names":["json-glib-1.0"],"versions":["1.10.8-1","1.10.6-1","1.10.0-1","1.6.6-2","1.6.6-1"]},"jsoncpp":{"dependency_names":["jsoncpp"],"versions":["1.9.8-1","1.9.7-1","1.9.6-1","1.9.5-2","1.9.5-1","1.8.4-1"]},"kafel":{"dependency_names":["kafel"],"program_names":["dump_policy_bpf"],"versions":["20231004-1"]},"kraken-engine":{"dependency_names":["kraken-engine"],"versions":["0.0.11-1","0.0.9-1","0.0.8-1","0.0.7-1","0.0.6-1","0.0.5-1","0.0.4-1","0.0.3-1","0.0.2-1","0.0.1-1"]},"lame":{"dependency_names":["mp3lame","mpglib"],"versions":["3.100-11","3.100-10","3.100-9","3.100-8","3.100-7","3.100-6","3.100-5","3.100-4","3.100-3","3.100-2","3.100-1","3.99.5-1"]},"lcms2":{"dependency_names":["lcms2"],"versions":["2.19.1-1","2.19-1","2.18-1","2.17-1","2.16-1","2.15-1","2.13.1-1","2.12-2","2.12-1"]},"leptonica":{"dependency_names":["lept"],"versions":["1.87.0-1","1.84.1-2","1.84.1-1","1.84.0-1","1.83.1-2","1.83.1-1"]},"libaegis":{"dependency_names":["libaegis"],"versions":["0.4.0-1"]},"libarchive":{"dependency_names":["libarchive"],"program_names":["bsdcat","bsdcpio","bsdtar"],"versions":["3.8.8-1","3.8.7-2","3.8.7-1","3.8.6-1","3.8.5-1","3.8.4-2","3.8.4-1","3.8.3-1","3.8.1-2","3.8.1-1","3.7.9-1","3.7.8-1","3.7.7-2","3.7.7-1","3.7.6-1","3.7.5-1","3.7.4-1","3.7.3-1","3.7.2-3","3.7.2-2","3.7.2-1","3.7.1-2","3.7.1-1","3.7.0-1","3.6.2-3","3.6.2-2","3.6.2-1"]},"libcap":{"dependency_names":["libcap"],"versions":["2.77-1","2.76-3","2.76-2","2.76-1"]},"libcbor":{"dependency_names":["libcbor"],"versions":["0.13.0-1"]},"libccp4c":{"dependency_names":["libccp4c"],"versions":["8.0.0-3","8.0.0-2","8.0.0-1","6.5.1-2","6.5.1-1"]},"libdicom":{"dependency_names":["libdicom"],"versions":["1.3.0-1","1.2.1-1","1.2.0-1","1.1.0-1","1.0.5-1","1.0.2-1","1.0.1-1"]},"libdrm":{"dependency_names":["libdrm","libdrm_amdgpu","libdrm_etnaviv","libdrm_exynos","libdrm_freedreno","libdrm_intel","libdrm_nouveau","libdrm_omap","libdrm_radeon","libdrm_tegra"],"versions":["2.4.134-1","2.4.133-1","2.4.131-1","2.4.130-1","2.4.129-1","2.4.128-1","2.4.127-1","2.4.126-1","2.4.125-1","2.4.124-1","2.4.123-1","2.4.121-1","2.4.115-1","2.4.114-1","2.4.111-1","2.4.110-1"]},"libebml":{"dependency_names":["libebml"],"versions":["1.4.5-3","1.4.5-2","1.4.5-1","1.4.4-2","1.4.4-1","1.4.3-1","1.4.2-1"]},"libexif":{"dependency_names":["libexif"],"versions":["0.6.25-1","0.6.24-5","0.6.24-4","0.6.24-3","0.6.24-2","0.6.24-1","0.6.22-1"]},"libffi":{"dependency_names":["libffi"],"versions":["3.6.0-1","3.5.2-2","3.5.2-1","3.5.1-2","3.5.1-1","3.4.8-1","3.4.6-3","3.4.6-2","3.4.6-1","3.4.4-4","3.4.4-3","3.4.4-2","3.4.4-1"]},"libffmpegthumbnailer":{"dependency_names":["libffmpegthumbnailer"],"versions":["2.2.4-1","2.2.3-2","2.2.3-1","2.2.2-4","2.2.2-3","2.2.2-2","2.2.2-1"]},"libgpiod":{"dependency_names":["libgpiod","libgpiodcxx"],"versions":["1.6.5-4","1.6.5-3","1.6.5-2","1.6.5-1","1.6.3-2","1.6.3-1"]},"libgrapheme":{"dependency_names":["libgrapheme"],"versions":["2.0.2-2","2.0.2-1"]},"libjpeg-turbo":{"dependency_names":["libjpeg","libturbojpeg"],"versions":["3.1.4.1-2","3.1.4.1-1","3.1.3-1","3.1.2-1","3.1.1-1","3.1.0-2","3.1.0-1","3.0.4-2","3.0.4-1","3.0.3-2","3.0.3-1","3.0.2-1","3.0.1-1","3.0.0-5","3.0.0-4","3.0.0-3","3.0.0-2","3.0.0-1","2.1.5.1-1","2.1.4-1","2.1.3-1","2.1.2-2","2.1.2-1","2.1.0-2","2.1.0-1"]},"libkqueue":{"dependency_names":["libkqueue"],"versions":["2.6.3-1","2.6.2-3","2.6.2-2","2.6.2-1","2.6.1-2","2.6.1-1"]},"liblangtag":{"dependency_names":["liblangtag"],"versions":["0.6.7-1","0.6.4-2","0.6.4-1","0.6.3-1"]},"liblastfm":{"dependency_names":["liblastfm"],"versions":["1.0.1-1"]},"liblbfgs":{"dependency_names":["liblbfgs"],"versions":["1.10-3","1.10-2","1.10-1"]},"libliftoff":{"dependency_names":["libliftoff"],"versions":["0.5.0-1","0.3.0-1"]},"liblzma":{"dependency_names":["liblzma"],"versions":["5.2.12-3","5.2.12-2","5.2.12-1","5.2.11-2","5.2.11-1","5.2.10-1","5.2.8-2","5.2.8-1","5.2.7-1","5.2.6-3","5.2.6-2","5.2.6-1","5.2.5-2","5.2.5-1","5.2.1-6","5.2.1-5","5.2.1-4","5.2.1-2"]},"libmatroska":{"dependency_names":["libmatroska"],"versions":["1.7.1-5","1.7.1-4","1.7.1-3","1.7.1-2","1.7.1-1","1.7.0-1","1.6.3-1"]},"libmicrohttpd":{"dependency_names":["libmicrohttpd"],"versions":["0.9.77-5","0.9.77-4","0.9.77-3","0.9.77-2","0.9.77-1","0.9.76-3","0.9.76-2","0.9.76-1"]},"libmysofa":{"dependency_names":["libmysofa"],"versions":["1.3.2-1"]},"libnpupnp":{"dependency_names":["libnpupnp"],"versions":["6.3.0-1","6.2.3-1","6.2.1-1","6.2.0-1","6.1.3-1","6.1.2-1","6.0.5-1","5.1.1-1","5.0.2-1","5.0.1-1","5.0.0-2","5.0.0-1","4.2.2-2","4.2.2-1","4.2.1-2","4.2.1-1"]},"libobsd":{"dependency_names":["libbsd-overlay","libobsd"],"versions":["1.1.1-1","1.1.0-1","1.0.0-1"]},"libopenjp2":{"dependency_names":["libopenjp2","libopenjpip"],"versions":["2.5.4-1","2.5.3-2","2.5.3-1","2.5.2-3","2.5.2-2","2.5.2-1","2.5.0-3","2.5.0-2","2.5.0-1","2.3.1-9","2.3.1-8","2.3.1-7","2.3.1-6","2.3.1-5","2.3.1-4","2.3.1-3","2.3.1-1"]},"libpfm":{"dependency_names":["libpfm"],"versions":["4.13.0-2","4.13.0-1"]},"libpng":{"dependency_names":["libpng"],"versions":["1.6.58-1","1.6.57-1","1.6.56-1","1.6.55-1","1.6.54-1","1.6.53-2","1.6.53-1","1.6.52-1","1.6.51-1","1.6.50-2","1.6.50-1","1.6.49-1","1.6.48-1","1.6.47-1","1.6.46-1","1.6.45-1","1.6.44-1","1.6.43-2","1.6.43-1","1.6.42-1","1.6.41-1","1.6.40-1","1.6.39-3","1.6.39-2","1.6.39-1","1.6.37-5","1.6.37-4","1.6.37-3","1.6.37-2","1.6.37-1","1.6.35-6","1.6.35-5","1.6.35-4","1.6.34-3","1.6.34-2","1.6.34-1","1.6.17-8","1.6.17-7","1.6.17-6","1.6.17-5","1.6.17-4","1.6.17-2"]},"libpsl":{"dependency_names":["libpsl"],"versions":["0.22.0-1","0.21.5-2","0.21.5-1","0.21.2-1"]},"libsass":{"dependency_names":["libsass"],"versions":["3.6.4-2","3.6.4-1"]},"libserialport":{"dependency_names":["libserialport"],"versions":["0.1.2-1"]},"libsigcplusplus-3":{"dependency_names":["sigc++-3.0"],"versions":["3.6.0-1"]},"libsignal-protocol-c":{"dependency_names":["libsignal-protocol-c"],"versions":["2.3.3-1"]},"libsndfile":{"dependency_names":["sndfile"],"versions":["1.2.2-2","1.2.2-1","1.2.0-1","1.1.0-6","1.1.0-5","1.1.0-4","1.1.0-3","1.1.0-2","1.1.0-1"]},"libsrtp2":{"dependency_names":["libsrtp2"],"versions":["2.8.0-1","2.7.0-1","2.6.0-1","2.5.0-1","2.4.2-1","2.4.0-1","2.2.0-1"]},"libssh2":{"dependency_names":["libssh2"],"versions":["1.11.1-2","1.11.1-1","1.11.0-1","1.10.0-5","1.10.0-4","1.10.0-3","1.10.0-2","1.10.0-1"]},"libtiff":{"dependency_names":["libtiff-4"],"versions":["4.7.1-5","4.7.1-4","4.7.1-3","4.7.1-2","4.7.1-1","4.7.0-1","4.6.0-1","4.5.1-1","4.5.0-3","4.5.0-2","4.5.0-1","4.4.0-3","4.4.0-2","4.4.0-1","4.1.0-5","4.1.0-4","4.1.0-3","4.1.0-2","4.1.0-1"]},"libtirpc":{"dependency_names":["libtirpc"],"versions":["1.3.7-1","1.3.3-1"]},"libtomcrypt":{"dependency_names":["libtomcrypt"],"versions":["1.18.2-1","1.17-1"]},"libunibreak":{"dependency_names":["libunibreak"],"versions":["6.1-1","5.1-3","5.1-2","5.1-1"]},"libupnp":{"dependency_names":["libupnp"],"versions":["1.14.27-1","1.14.25-3","1.14.25-2","1.14.25-1","1.14.24-1","1.14.23-1","1.14.20-2","1.14.20-1","1.14.19-1","1.14.18-1","1.14.17-1","1.14.15-1","1.14.14-2","1.14.14-1","1.14.13-1","1.14.12-2","1.14.12-1"]},"liburing":{"dependency_names":["liburing"],"versions":["2.14-1","2.12-1","2.5-2","2.5-1","2.4-2","2.4-1","2.3-3","2.3-2","2.3-1","2.2-2","2.2-1","2.1-1","2.0-1"]},"libusb":{"dependency_names":["libusb-1.0"],"versions":["1.0.29-2","1.0.29-1","1.0.28-1","1.0.27-3","1.0.27-2","1.0.27-1","1.0.26-5","1.0.26-4","1.0.26-3","1.0.26-2","1.0.26-1"]},"libuv":{"dependency_names":["libuv"],"versions":["1.51.0-1","1.49.1-1","1.48.0-1","1.47.0-1","1.46.0-1","1.44.2-2","1.44.2-1","1.44.1-1","1.43.0-1","1.42.0-1","1.41.0-1","1.18.0-3","1.18.0-2","1.18.0-1"]},"libwebp":{"dependency_names":["libsharyuv","libwebp","libwebpdecoder","libwebpdemux","libwebpmux"],"program_names":["cwebp","dwebp","webpinfo","webpmux"],"versions":["1.6.0-1","1.5.0-1","1.3.2-2","1.3.2-1","1.3.1-2","1.3.1-1"]},"libwebsockets":{"dependency_names":["libwebsockets"],"versions":["4.3.6-1","4.3.5-1","4.3.3-1","4.3.2-3","4.3.2-2","4.3.2-1","4.0.21-2","4.0.21-1","4.0.13-1"]},"libxcursor":{"dependency_names":["xcursor"],"versions":["1.2.3-1","1.2.1-1"]},"libxext":{"dependency_names":["xext"],"versions":["1.3.7-1","1.3.6-1","1.3.5-1","1.3.4-1"]},"libxinerama":{"dependency_names":["xinerama"],"versions":["1.1.6-1","1.1.5-1","1.1.4-1"]},"libxkbcommon":{"dependency_names":["xkbcommon","xkbcommon-x11","xkbregistry"],"versions":["1.11.0-2","1.11.0-1","1.10.0-1","1.9.2-1","1.9.1-1","1.8.1-1","1.7.0-1"]},"libxml2":{"dependency_names":["libxml-2.0"],"versions":["2.15.3-1","2.15.2-1","2.15.1-1","2.15.0-1","2.14.6-1","2.14.5-1","2.14.4-1","2.14.3-1","2.14.2-1","2.14.1-1","2.13.6-1","2.13.5-1","2.13.4-1","2.13.3-1","2.13.2-1","2.13.1-1","2.13.0-1","2.12.7-1","2.12.6-1","2.12.5-1","2.11.6-3","2.11.6-2","2.11.6-1","2.11.5-2","2.11.5-1","2.11.4-1","2.11.1-1","2.10.4-1","2.10.3-4","2.10.3-3","2.10.3-2","2.10.3-1","2.10.2-2","2.10.2-1","2.9.14-1","2.9.7-14","2.9.7-13","2.9.7-12","2.9.7-11","2.9.7-10","2.9.7-9","2.9.7-8","2.9.7-7","2.9.7-6","2.9.7-5","2.9.7-4","2.9.7-3","2.9.7-2","2.9.7-1","2.9.2-5","2.9.2-4","2.9.2-2"]},"libxmlpp":{"dependency_names":["libxml++-5.0"],"versions":["5.6.1-1","5.6.0-1","5.4.0-2","5.4.0-1","5.2.0-1","5.0.3-1","5.0.2-2","5.0.2-1","5.0.1-1","3.0.1-2","3.0.1-1"]},"libxrandr":{"dependency_names":["xrandr"],"versions":["1.5.5-1","1.5.4-1","1.5.2-1"]},"libxrender":{"dependency_names":["xrender"],"versions":["0.9.12-1","0.9.10-1"]},"libxslt":{"dependency_names":["libxslt","libexslt"],"program_names":["xsltproc"],"versions":["1.1.45-1","1.1.43-3","1.1.43-2","1.1.43-1","1.1.39-1","1.1.38-1","1.1.37-4","1.1.37-3","1.1.37-2","1.1.37-1","1.1.36-1","1.1.35-1","1.1.34-1"]},"libxv":{"dependency_names":["xv"],"versions":["1.0.13-1","1.0.11-1"]},"libxxf86vm":{"dependency_names":["xxf86vm"],"versions":["1.1.7-1","1.1.6-1","1.1.4-1"]},"libyaml":{"dependency_names":["yaml-0.1"],"versions":["0.2.5-1"]},"littlefs":{"dependency_names":["littlefs"],"versions":["2.11.2-1","2.11.0-1","2.10.2-1","2.10.1-1","2.9.3-1","2.9.0-1"]},"lmdb":{"dependency_names":["lmdb"],"program_names":["mdb_stat","mdb_copy","mdb_dump","mdb_load"],"versions":["0.9.33-1","0.9.29-3","0.9.29-2","0.9.29-1"]},"lua":{"dependency_names":["lua-5.5","lua"],"versions":["5.5.0-1","5.4.8-1","5.4.6-5","5.4.6-4","5.4.6-3","5.4.6-2","5.4.6-1","5.4.4-1","5.4.3-2","5.4.3-1","5.3.6-2","5.3.6-1","5.3.0-6","5.3.0-5","5.3.0-4","5.3.0-2"]},"luajit":{"dependency_names":["luajit"],"program_names":["luajit"],"versions":["2.1.1720049189-3","2.1.1720049189-2","2.1.1720049189-1","2.1.1713773202-1"]},"ludocode-mpack":{"dependency_names":["ludocode-mpack"],"versions":["1.1.1-1","1.1-1","1.0-1"]},"lvgl":{"dependency_names":["lvgl","lvgl_demos","lvgl_examples","lvgl_thorvg"],"versions":["9.5.0-1","9.4.0-1","9.3.0-1","9.2.2-2","9.2.2-1"]},"lz4":{"dependency_names":["liblz4"],"versions":["1.10.0-2","1.10.0-1","1.9.4-2","1.9.4-1","1.9.3-1","1.9.2-1"]},"lzo2":{"dependency_names":["lzo2"],"versions":["2.10-4","2.10-3","2.10-2","2.10-1"]},"m4":{"program_names":["m4"],"versions":["1.4.19-2","1.4.19-1"]},"magic_enum":{"dependency_names":["magic_enum"],"versions":["0.9.8-1","0.9.7-1","0.9.6-1","0.9.5-1","0.9.3-1","0.9.2-1","0.8.2-1","0.8.1-1"]},"mdds":{"dependency_names":["mdds-2.1"],"versions":["2.1.1-1","2.0.1-1","1.6.0-2","1.6.0-1"]},"microsoft-gsl":{"dependency_names":["msft_gsl"],"versions":["4.2.1-1","4.2.0-1","4.0.0-1","3.1.0-1","2.0.0-1"]},"mimalloc":{"dependency_names":["mimalloc"],"versions":["3.3.2-1","3.2.7-1","3.1.5-1"]},"miniaudio":{"dependency_names":["miniaudio"],"versions":["0.11.22-2","0.11.22-1","0.11.21-2","0.11.21-1"]},"minifycpp":{"dependency_names":["minifycpp"],"versions":["0.2.4-1","0.1.2-1"]},"miniz":{"dependency_names":["miniz"],"versions":["3.1.0-1","3.0.2-1","3.0.1-1","2.2.0-1","2.1.0-2","2.1.0-1"]},"minizip-ng":{"dependency_names":["minizip"],"versions":["4.0.10-3","4.0.10-2","4.0.10-1","4.0.9-2","4.0.9-1","4.0.7-1","4.0.4-1","4.0.1-2","4.0.1-1","4.0.0-2","4.0.0-1","3.0.10-1","3.0.8-1","3.0.7-1","3.0.6-1"]},"mldsa-native":{"dependency_names":["mldsa-native"],"versions":["1.0.0-1"]},"mocklibc":{"versions":["1.0-2"]},"mpark-patterns":{"dependency_names":["mpark_patterns"],"versions":["0.3.0-1"]},"mpdecimal":{"dependency_names":["mpdec","mpdecpp"],"versions":["2.5.1-4","2.5.1-3","2.5.1-2","2.5.1-1","2.5.0-2","2.5.0-1"]},"msgpackc-cxx":{"dependency_names":["msgpack-cxx"],"versions":["7.0.0-1","6.1.1-2","6.1.1-1","6.1.0-2","6.1.0-1","5.0.0-1"]},"mt32emu":{"dependency_names":["mt32emu"],"versions":["2.7.2-1","2.7.1-1","2.7.0-1","2.6.1-1","2.5.3-1","2.5.0-1","2.4.2-2","2.4.2-1"]},"muellan-clipp":{"versions":["1.2.3-1"]},"mujs":{"dependency_names":["mujs"],"program_names":["mujs","mujs-pp"],"versions":["1.3.8-1","1.3.6-1","1.3.5-1","1.3.3-1"]},"nanoarrow":{"dependency_names":["nanoarrow","nanoarrow-ipc"],"versions":["0.8.0-1","0.7.0-1","0.6.0-1","0.5.0-1"]},"nanobind":{"dependency_names":["nanobind"],"versions":["2.12.0-1","2.10.2-1","2.9.2-1","2.8.0-1","2.7.0-4","2.7.0-3","2.7.0-2","2.7.0-1","2.4.0-2","2.4.0-1","2.2.0-2","2.2.0-1","2.1.0-1"]},"nativefiledialog-extended":{"dependency_names":["nativefiledialog-extended"],"versions":["1.2.1-1","1.1.1-2","1.1.1-1"]},"netstring-c":{"dependency_names":["netstring-c"],"versions":["0.0.0-4","0.0.0-3","0.0.0-2","0.0.0-1"]},"nghttp2":{"dependency_names":["libnghttp2"],"versions":["1.62.1-3","1.62.1-2","1.62.1-1","1.58.0-1","1.56.0-1"]},"nlohmann_json":{"dependency_names":["nlohmann_json"],"versions":["3.12.0-1","3.11.3-1","3.11.2-1","3.10.5-1","3.9.1-1","3.7.0-1","3.4.0-2","3.4.0-1","3.3.0-1"]},"nng":{"dependency_names":["nng"],"versions":["1.11-1","1.5.2-4","1.5.2-3","1.5.2-2","1.5.2-1","1.3.2-2","1.3.2-1"]},"nonstd-any-lite":{"versions":["0.2.0-1"]},"nonstd-byte-lite":{"versions":["0.2.0-1"]},"nonstd-expected-lite":{"dependency_names":["expected-lite"],"versions":["0.3.0-2","0.3.0-1"]},"nonstd-observer-ptr-lite":{"versions":["0.4.0-1"]},"nonstd-optional-lite":{"versions":["3.2.0-1"]},"nonstd-span-lite":{"versions":["0.5.0-1"]},"nonstd-status-value-lite":{"versions":["1.1.0-1"]},"nonstd-string-view-lite":{"versions":["1.3.0-1"]},"nowide":{"dependency_names":["nowide"],"versions":["11.3.0-1","11.2.0-1"]},"oatpp":{"dependency_names":["oatpp","oatpp-test"],"versions":["1.3.1-2","1.3.1-1","1.3.0-2","1.3.0-1"]},"oatpp-openssl":{"dependency_names":["oatpp-openssl"],"versions":["1.3.0-2","1.3.0-1"]},"oatpp-sqlite":{"dependency_names":["oatpp-sqlite"],"versions":["1.3.0-1"]},"oatpp-swagger":{"dependency_names":["oatpp-swagger"],"versions":["1.3.0-1"]},"oatpp-websocket":{"dependency_names":["oatpp-websocket"],"versions":["1.3.0-1"]},"oatpp-zlib":{"dependency_names":["oatpp-zlib"],"versions":["1.3.0-1"]},"ogg":{"dependency_names":["ogg"],"versions":["1.3.6-1","1.3.5-6","1.3.5-5","1.3.5-4","1.3.5-3","1.3.5-2","1.3.5-1","1.3.2-7","1.3.2-6","1.3.2-5","1.3.2-4","1.3.2-2"]},"onqtam-doctest":{"versions":["2.4.0-1","2.3.7-1"]},"openal-soft":{"dependency_names":["openal"],"versions":["1.24.3-2","1.24.3-1","1.24.2-1","1.24.1-1","1.24.0-1","1.23.1-3","1.23.1-2","1.23.1-1","1.23.0-1","1.22.2-8","1.22.2-7","1.22.2-6","1.22.2-5","1.22.2-4","1.22.2-3","1.22.2-2","1.22.2-1","1.21.0-1"]},"openblas":{"dependency_names":["openblas"],"versions":["0.3.28-2","0.3.28-1"]},"opencl-headers":{"dependency_names":["opencl-headers"],"versions":["2024.10.24-1","2023.04.17-1","2022.05.18-1","2021.06.30-1"]},"openh264":{"dependency_names":["openh264"],"versions":["2.6.0-1","2.5.0-1","2.4.1-1","2.3.1-1","1.7.0-1"]},"openssl":{"dependency_names":["libcrypto","libssl","openssl"],"versions":["3.0.8-3","3.0.8-2","3.0.8-1","3.0.7-2","3.0.7-1","3.0.2-1","1.1.1l-3","1.1.1l-2","1.1.1l-1","1.1.1k-2","1.1.1k-1"]},"opus":{"dependency_names":["opus"],"versions":["1.6.1-1","1.6-1","1.5.2-1","1.4-1"]},"orcus":{"dependency_names":["liborcus-0.19"],"versions":["0.19.2-1","0.17.2-3","0.17.2-2","0.17.2-1"]},"orocos_kdl":{"dependency_names":["orocos-kdl"],"versions":["1.5.1-1"]},"pango":{"dependency_names":["pango","pangoft2","pangoxft","pangowin32","pangocairo"],"versions":["1.56.4-1","1.56.3-1","1.56.2-1","1.56.1-1","1.56.0-1","1.51.0-2","1.51.0-1"]},"pcg":{"versions":["0.98.1-2","0.98.1-1"]},"pciaccess":{"dependency_names":["pciaccess"],"versions":["0.19-1","0.18.1-2","0.18.1-1","0.18-1"]},"pcre":{"dependency_names":["libpcre","libpcreposix"],"versions":["8.45-5","8.45-4","8.45-3","8.45-2","8.45-1","8.37-4","8.37-3","8.37-2","8.37-1"]},"pcre2":{"dependency_names":["libpcre2-8","libpcre2-16","libpcre2-32","libpcre2-posix"],"versions":["10.47-3","10.47-2","10.47-1","10.46-1","10.45-4","10.45-3","10.45-2","10.45-1","10.44-2","10.44-1","10.43-1","10.42-5","10.42-4","10.42-3","10.42-2","10.42-1","10.40-3","10.40-2","10.40-1","10.39-3","10.39-2","10.39-1","10.23-1"]},"pegtl":{"dependency_names":["pegtl"],"versions":["3.2.8-1","3.2.7-1"]},"physfs":{"dependency_names":["physfs"],"versions":["3.2.0-2","3.2.0-1"]},"pixman":{"dependency_names":["pixman-1"],"versions":["0.46.4-1","0.46.2-1","0.46.0-1","0.44.2-1","0.44.0-1","0.43.4-1","0.43.2-1","0.42.2-1"]},"pkgconf":{"dependency_names":["libpkgconf"],"versions":["2.5.1-1","2.5.0-1","2.4.3-1","2.3.0-1","2.1.1-1","1.9.3-2","1.9.3-1","1.9.2-1","1.9.0-1"]},"protobuf":{"dependency_names":["protobuf-lite","protobuf","protoc"],"program_names":["protoc"],"versions":["25.2-4","25.2-3","25.2-2","25.2-1","3.21.12-5","3.21.12-4","3.21.12-3","3.21.12-2","3.21.12-1","3.21.9-1","3.20.1-1","3.20.0-1","3.12.2-2","3.12.2-1","3.5.1-3","3.5.1-2","3.5.0-4","3.5.0-3","3.5.0-2"]},"protobuf-c":{"dependency_names":["libprotobuf-c"],"versions":["1.5.0-1","1.4.1-3","1.4.1-2","1.4.1-1","1.3.0-1"]},"proxy-libintl":{"dependency_names":["intl"],"versions":["0.5-1","0.4-1"]},"pugixml":{"dependency_names":["pugixml"],"versions":["1.15-1","1.14-1","1.13-3","1.13-2","1.13-1","1.12.1-1","1.12-1","1.11.4-1"]},"pv":{"program_names":["pv"],"versions":["1.10.5-1","1.10.3-1","1.10.2-1","1.9.34-2","1.9.34-1","1.9.31-1","1.8.14-1","1.8.13-1","1.8.12-1","1.8.10-1","1.8.9-1","1.8.5-1"]},"pybind11":{"dependency_names":["pybind11"],"versions":["3.0.0-1","2.13.5-1","2.13.1-1","2.13.0-1","2.12.0-1","2.11.1-1","2.10.4-1","2.10.3-1","2.10.0-1","2.9.0-1","2.6.1-1","2.6.0-1","2.3.0-2","2.3.0-1","2.2.4-1","2.2.3-1","2.2.2-1"]},"qarchive":{"dependency_names":["qarchive"],"versions":["2.2.7-1","2.2.6-1","2.2.4-1","2.2.3-1","2.2.2-1","2.2.1-2","2.2.1-1","2.2.0-1","2.0.2-1"]},"qrencode":{"dependency_names":["libqrencode"],"versions":["4.1.1-3","4.1.1-2","4.1.1-1"]},"quazip":{"dependency_names":["quazip"],"versions":["1.4-1","1.3-1","0.7.3-1"]},"quickjs-ng":{"dependency_names":["quickjs-ng"],"program_names":["qjsc","qjs"],"versions":["0.15.1-1","0.15.0-1","0.14.0-1","0.13.0-1","0.12.1-1","0.12.0-1","0.11.0-1","0.10.1-1","0.10.0-1","0.9.0-1"]},"quill":{"dependency_names":["quill"],"versions":["12.0.0-1","11.1.0-1","11.0.2-1","11.0.1-1","10.2.0-1","10.1.0-1","10.0.1-1","10.0.0-1","9.0.3-1","9.0.2-1","9.0.0-1","8.2.0-1","8.1.1-1","8.1.0-1","8.0.0-1","7.5.0-1","7.4.0-1","7.3.0-1","7.2.2-1","7.1.0-1","7.0.0-1","6.0.0-1","4.5.0-1","4.4.0-1","4.2.0-1"]},"rang":{"dependency_names":["rang"],"versions":["3.2-1","2.0-1"]},"range-v3":{"dependency_names":["range-v3"],"versions":["0.12.0-2","0.12.0-1","0.11.0-1","0.10.0-1","0.3.6-1"]},"rapidjson":{"dependency_names":["rapidjson"],"versions":["1.1.0-2","1.1.0-1"]},"rdkafka":{"dependency_names":["rdkafka","rdkafka++"],"versions":["2.10.0-1","2.5.3-1","2.3.0-1","2.1.0-2","2.1.0-1","1.9.2-5","1.9.2-4","1.9.2-3","1.9.2-2","1.9.2-1","1.9.0-2","1.9.0-1","1.8.2-1","1.7.0-1","1.5.2-1","1.5.0-1","1.4.4-1","1.4.0-1"]},"re2":{"dependency_names":["re2"],"versions":["20230301-3","20230301-2","20230301-1","20220401-1","20201101-1"]},"reflex":{"dependency_names":["reflex"],"program_names":["reflex"],"versions":["5.1.1-3","5.1.1-2","5.1.1-1","5.1.0-2","5.1.0-1","3.2.11-1","3.2.7-1","3.2.3-1","3.0.1-2","3.0.1-1"]},"robin-map":{"dependency_names":["robin-map"],"versions":["1.4.0-1","1.3.0-1","1.2.1-1"]},"rtaudio":{"dependency_names":["rtaudio"],"versions":["6.0.1-1","5.2.0-1"]},"rubberband":{"dependency_names":["rubberband"],"versions":["4.0.0-1","3.3.0-1","2.0.2-1"]},"rxcpp":{"dependency_names":["rxcpp"],"versions":["4.1.1-1","4.1.0-1"]},"s2n-tls":{"dependency_names":["s2n-tls"],"versions":["1.5.27-1"]},"sassc":{"program_names":["sassc"],"versions":["3.6.2-2","3.6.2-1"]},"scdoc":{"program_names":["scdoc"],"versions":["1.11.4-1"]},"sdl2":{"dependency_names":["sdl2","sdl2main","sdl2_test"],"versions":["2.32.8-1","2.30.6-2","2.30.6-1","2.30.3-3","2.30.3-2","2.30.3-1","2.28.5-2","2.28.5-1","2.28.1-2","2.28.1-1","2.26.5-5","2.26.5-4","2.26.5-3","2.26.5-2","2.26.5-1","2.26.0-2","2.26.0-1","2.24.2-1","2.24.1-4","2.24.1-3","2.24.1-2","2.24.1-1","2.24.0-5","2.24.0-4","2.24.0-3","2.24.0-2","2.24.0-1","2.0.20-6","2.0.20-5","2.0.20-4","2.0.20-3","2.0.20-2","2.0.20-1","2.0.18-2","2.0.18-1","2.0.12-3","2.0.12-2","2.0.12-1","2.0.3-6","2.0.3-5","2.0.3-4"]},"sdl2_image":{"dependency_names":["sdl2_image"],"versions":["2.6.3-3","2.6.3-2","2.6.3-1","2.6.2-1","2.0.5-3","2.0.5-2","2.0.5-1"]},"sdl2_mixer":{"dependency_names":["sdl2_mixer"],"versions":["2.6.2-4","2.6.2-3","2.6.2-2","2.6.2-1","2.0.4-3","2.0.4-2","2.0.4-1"]},"sdl2_net":{"dependency_names":["sdl2_net"],"versions":["2.2.0-3","2.2.0-2","2.2.0-1","2.0.1-4","2.0.1-3","2.0.1-2","2.0.1-1"]},"sdl2_ttf":{"dependency_names":["sdl2_ttf"],"versions":["2.20.1-4","2.20.1-3","2.20.1-2","2.20.1-1","2.0.12-3","2.0.12-2","2.0.12-1"]},"sdl3":{"dependency_names":["sdl3"],"versions":["3.4.2-1","3.4.0-2","3.4.0-1","3.2.10-2","3.2.10-1","3.2.4-3","3.2.4-2"]},"sdl3_image":{"dependency_names":["sdl3_image"],"versions":["3.2.4-1"]},"sdl3_ttf":{"dependency_names":["sdl3_ttf"],"versions":["3.2.2-2"]},"sds":{"dependency_names":["sds"],"versions":["2.0.0-2","2.0.0-1"]},"sergiusthebest-plog":{"dependency_names":["plog"],"versions":["1.1.10-2","1.1.10-1","1.1.4-1"]},"sfml":{"dependency_names":["sfml","sfml-all"],"versions":["3.0.2-1","3.0.1-1","2.6.2-2","2.6.2-1","2.6.1-1","2.6.0-2","2.6.0-1","2.5.1-4","2.5.1-3","2.5.1-2","2.5.1-1"]},"simdjson":{"dependency_names":["simdjson"],"versions":["4.2.2-1","3.13.0-1","3.12.3-1","3.12.2-1","3.11.3-1","3.10.1-1","3.3.0-2","3.3.0-1","3.1.1-1"]},"slirp":{"dependency_names":["slirp"],"versions":["4.9.3-1","4.9.1-1","4.9.0-1","4.8.0-1","4.7.0-1","4.6.1-2","4.6.1-1"]},"snitch":{"dependency_names":["snitch"],"versions":["1.3.2-1"]},"soundtouch":{"dependency_names":["soundtouch"],"versions":["2.4.1-1","2.4.0-1","2.3.2-5","2.3.2-4","2.3.2-3","2.3.2-2","2.3.2-1"]},"sparkcli":{"dependency_names":["sparkcli"],"versions":["0.0.6-1","0.0.4-1","0.0.1-1"]},"sparrow":{"dependency_names":["sparrow"],"versions":["0.0.2-1"]},"sparsehash-c11":{"dependency_names":["sparsehash-c11"],"versions":["2.11.1-1"]},"spdlog":{"dependency_names":["spdlog"],"versions":["1.17.0-1","1.15.3-5","1.15.3-4","1.15.3-3","1.15.3-2","1.15.3-1","1.15.2-3","1.15.2-2","1.15.2-1","1.15.1-1","1.15.0-1","1.14.1-2","1.14.1-1","1.14.0-1","1.13.0-4","1.13.0-3","1.13.0-2","1.13.0-1","1.12.0-2","1.12.0-1","1.11.0-2","1.11.0-1","1.10.0-3","1.10.0-2","1.10.0-1","1.9.2-1","1.8.5-1","1.8.2-1","1.8.1-1","1.8.0-1","1.4.2-1","1.3.1-1","1.1.0-1","0.17.0-2","0.17.0-1","0.16.3-1","0.11.0-1"]},"speexdsp":{"dependency_names":["speexdsp"],"versions":["1.2.1-8","1.2.1-7","1.2.1-6","1.2.1-5","1.2.1-4","1.2.1-3","1.2.1-2","1.2.1-1","1.2.0-1"]},"spng":{"dependency_names":["spng"],"versions":["0.7.4-1","0.7.2-1","0.7.0-1","0.5.0-1","0.4.5-1","0.4.4-1","0.4.3-1","0.4.2-2","0.4.2-1"]},"sqlite3":{"dependency_names":["sqlite3"],"versions":["3.53.3-1","3.53.2-1","3.53.1-1","3.53.0-1","3.52.0-1","3.51.2-1","3.51.1-1","3.51.0-1","3.50.4-1","3.50.3-1","3.50.2-1","3.50.1-1","3.50.0-1","3.49.2-1","3.49.1-1","3.49.0-1","3.48.0-1","3.47.2-2","3.47.2-1","3.47.1-1","3.47.0-1","3.46.1-3","3.46.1-2","3.46.1-1","3.46.0-1","3.45.3-1","3.45.2-1","3.45.1-1","3.45.0-1","3.44.2-1","3.44.1-1","3.44.0-1","3.43.2-1","3.43.1-2","3.43.1-1","3.43.0-1","3.42.0-1","3.41.2-2","3.41.2-1","3.41.1-1","3.41.0-1","3.40.0-1","3.39.4-2","3.39.4-1","3.39.3-1","3.38.0-1","3.34.1-1"]},"sqlitecpp":{"dependency_names":["sqlitecpp"],"versions":["3.3.2-1","3.3.1-1"]},"sqlpp11":{"dependency_names":["sqlpp11"],"program_names":["ddl2cpp"],"versions":["0.64-1"]},"stc":{"dependency_names":["stc"],"program_names":["checkscoped"],"versions":["5.0-2","5.0-1"]},"stduuid":{"dependency_names":["stduuid"],"versions":["1.2.3-2","1.2.3-1"]},"svtjpegxs":{"dependency_names":["SvtJpegxs"],"versions":["0.9.0-2","0.9.0-1"]},"tabulate":{"dependency_names":["tabulate"],"versions":["1.5-1","1.4-2","1.4-1"]},"taglib":{"dependency_names":["taglib"],"versions":["2.1.1-1","2.0.2-1","2.0-1","1.13.1-1","1.13-4","1.13-3","1.13-2","1.13-1","1.12-3","1.12-2","1.12-1"]},"tclap":{"dependency_names":["tclap"],"versions":["1.2.4-2","1.2.4-1","1.2.2-1","1.2.1-1"]},"termbox":{"dependency_names":["termbox"],"versions":["1.1.2-2","1.1.2-1"]},"termbox2":{"dependency_names":["termbox2"],"versions":["2.5.0-2","2.5.0-1"]},"theora":{"dependency_names":["theoraenc","theoradec","theora"],"versions":["1.2.0-1","1.1.1-6","1.1.1-5","1.1.1-4","1.1.1-3","1.1.1-2","1.1.1-1"]},"tinyfsm":{"dependency_names":["tinyfsm"],"versions":["0.3.3-2","0.3.3-1"]},"tinyply":{"dependency_names":["tinyply"],"versions":["2.3.4-1","2.2-2","2.2-1","2.1-2","2.1-1","2.0-1"]},"tinyxml2":{"dependency_names":["tinyxml2"],"versions":["11.0.0-2","11.0.0-1","10.1.0-1","10.0.0-1","9.0.0-1","7.0.1-1"]},"tkrzw":{"dependency_names":["tkrzw"],"program_names":["tkrzw_dbm_perf","tkrzw_dbm_util","tkrzw_ulog_util"],"versions":["1.0.32-1"]},"tl-expected":{"dependency_names":["tl-expected"],"versions":["1.3.1-1","1.1.0-1","1.0.0-1"]},"tl-optional":{"dependency_names":["tl-optional"],"versions":["1.1.0-1","1.0.0-1"]},"tomlplusplus":{"dependency_names":["tomlplusplus"],"versions":["3.4.0-1","3.3.0-1","3.2.0-1","3.1.0-1","3.0.0-1","2.5.0-1"]},"tracy":{"dependency_names":["tracy"],"versions":["0.13.0-1","0.12.2-1","0.10-1","0.9.1-1","0.9-1","0.8.2.1-1","0.8.1-1"]},"tree-sitter":{"dependency_names":["tree-sitter"],"versions":["0.26.3-1","0.22.5-3","0.22.5-2","0.22.5-1"]},"trompeloeil":{"dependency_names":["trompeloeil"],"versions":["49-1","48-1","47-1","39-1","38-1"]},"tronkko-dirent":{"versions":["1.23.2-1"]},"turtle":{"versions":["1.3.2-1","1.3.1-1"]},"type_safe":{"dependency_names":["type_safe"],"versions":["0.2.4-1","0.2.3-3","0.2.3-2","0.2.3-1"]},"uchardet":{"dependency_names":["uchardet"],"versions":["0.0.8-1","0.0.7-3","0.0.7-2","0.0.7-1"]},"unit-system":{"dependency_names":["unit-system"],"versions":["0.7.1-1","0.7.0-1","0.6.1-1","0.5.2-1","0.5.0-1"]},"unittest-cpp":{"dependency_names":["unittest-cpp"],"versions":["2.0.0-2","2.0.0-1"]},"unity":{"dependency_names":["unity"],"versions":["2.6.1-1","2.5.2-1"]},"usbbluetooth":{"dependency_names":["usbbluetooth"],"versions":["0.0.9-1"]},"utf8proc":{"dependency_names":["libutf8proc"],"versions":["2.10.0-1","2.9.0-1","2.8.0-1","2.7.0-1","2.6.0-1"]},"utfcpp":{"dependency_names":["utf8cpp"],"versions":["4.0.8-1","4.0.6-1","4.0.5-1","3.2.4-1","3.2.3-1","3.2.2-1","3.2.1-1","2.3.5-1"]},"uthash":{"dependency_names":["uthash"],"versions":["2.4.0-1","2.3.0-2","2.3.0-1"]},"vo-aacenc":{"dependency_names":["vo-aacenc"],"versions":["0.1.3-2","0.1.3-1"]},"vorbis":{"dependency_names":["vorbis","vorbisfile","vorbisenc"],"versions":["1.3.7-4","1.3.7-3","1.3.7-2","1.3.7-1","1.3.7-0","1.3.5-8","1.3.5-7","1.3.5-6","1.3.5-5","1.3.5-4","1.3.5-2"]},"vulkan-headers":{"dependency_names":["VulkanHeaders"],"versions":["1.4.346-1","1.3.283-1","1.3.265-1","1.2.203-1","1.2.158-2","1.2.158-1","1.2.142-1"]},"vulkan-memory-allocator":{"dependency_names":["VulkanMemoryAllocator"],"versions":["3.3.0-1","3.1.0-1","3.0.1-1"]},"vulkan-validationlayers":{"versions":["1.2.158-2","1.2.158-1"]},"wayland":{"dependency_names":["wayland-client","wayland-cursor","wayland-egl","wayland-server"],"program_names":["wayland-scanner"],"versions":["1.25.0-1","1.24.0-2","1.24.0-1","1.23.1-1","1.23.0-1","1.20.0-1"]},"wayland-protocols":{"dependency_names":["wayland-protocols"],"versions":["1.49-1","1.48-1","1.47-1","1.46-1","1.45-1","1.44-1","1.43-1","1.42-1","1.41-1","1.40-1","1.39-1","1.38-1","1.37-1","1.35-1","1.32-1","1.24-1"]},"websocketpp":{"dependency_names":["websocketpp"],"versions":["0.8.2-2","0.8.2-1"]},"win-iconv":{"dependency_names":["iconv"],"program_names":["win_iconv"],"versions":["0.0.9-1","0.0.8-1"]},"wren":{"dependency_names":["wren"],"versions":["0.4.0-3","0.4.0-2","0.4.0-1","0.3.0-1"]},"x-plane-sdk":{"dependency_names":["xplm","xpwidgets","xpcpp"],"versions":["4.2.0-1","4.0.1-1"]},"xdelta3":{"program_names":["xdelta3"],"versions":["3.2.0-1","3.1.0-1"]},"xtensor":{"dependency_names":["xtensor"],"versions":["0.21.5-1"]},"xtl":{"dependency_names":["xtl"],"versions":["0.8.0-1","0.6.12-1"]},"xvidcore":{"dependency_names":["xvidcore"],"versions":["1.3.7-1"]},"xxhash":{"dependency_names":["libxxhash"],"versions":["0.8.3-2","0.8.3-1","0.8.2-1","0.8.1-2","0.8.1-1","0.8.0-2","0.8.0-1","0.6.5-1"]},"yajl":{"dependency_names":["yajl"],"versions":["2.1.0-5","2.1.0-4","2.1.0-3","2.1.0-2","2.1.0-1"]},"yaml-cpp":{"dependency_names":["yaml-cpp"],"versions":["0.8.0-2","0.8.0-1"]},"yyjson":{"dependency_names":["yyjson"],"versions":["0.12.0-1"]},"zlib":{"dependency_names":["zlib"],"versions":["1.3.2-1","1.3.1-3","1.3.1-2","1.3.1-1","1.3-5","1.3-4","1.3-3","1.3-2","1.3-1","1.2.13-4","1.2.13-3","1.2.13-2","1.2.13-1","1.2.12-1","1.2.11-6","1.2.11-5","1.2.11-4","1.2.11-3","1.2.11-2","1.2.11-1","1.2.8-8","1.2.8-7","1.2.8-6","1.2.8-4","1.2.8-2"]},"zlib-ng":{"dependency_names":["zlib-ng"],"versions":["2.3.3-1","2.3.2-2","2.3.2-1","2.3.1-1","2.2.5-1","2.2.4-2","2.2.4-1","2.2.3-1","2.2.2-1"]},"zpp_bits":{"dependency_names":["zpp_bits"],"versions":["4.5.1-1","4.5-1","4.4.24-1","4.4.23-1"]},"zstd":{"dependency_names":["libzstd"],"versions":["1.5.7-3","1.5.7-2","1.5.7-1","1.5.6-2","1.5.6-1","1.5.5-1","1.5.4-1","1.4.5-1","1.3.3-2","1.3.3-1"]},"zycore":{"dependency_names":["zycore"],"versions":["1.5.2-1","1.5.1-1"]},"zydis":{"dependency_names":["zydis"],"versions":["4.1.1-1"]}} From fd7c21296f3c32ec16460f767bc34a68af85aa9c Mon Sep 17 00:00:00 2001 From: bcumming Date: Fri, 10 Jul 2026 13:57:59 +0200 Subject: [PATCH 20/29] bugfix: treat lack of credentials when deleting as a hard error --- src/cli/delete.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/cli/delete.cpp b/src/cli/delete.cpp index 98441da9..dbd841df 100644 --- a/src/cli/delete.cpp +++ b/src/cli/delete.cpp @@ -67,10 +67,14 @@ int image_delete([[maybe_unused]] const image_delete_args& args, if (auto c = resolve_registry_credentials(settings.calling_environment, artifactory_url, args.username, args.token)) { - if (!*c) { - term::error("full credentials must be provided", c.error()); + // resolve_registry_credentials returns nullopt when no credentials + // were found; deletion is never anonymous, so this is an error. + if (!c->has_value()) { + term::error("full credentials must be provided to delete a uenv: " + "see the --token and --username flags"); + return 1; } - credentials = (*c).value(); + credentials = c->value(); } else { term::error("{}", c.error()); return 1; From 2abb1bd8d9e41e2dbaba69eacc3767b05a573226 Mon Sep 17 00:00:00 2001 From: bcumming Date: Fri, 10 Jul 2026 13:58:52 +0200 Subject: [PATCH 21/29] bugfix: failure during download leaves no half state - use a temporary file for downloads --- src/oci/client.cpp | 48 +++++++++++++++++++++++++++++++------- test/unit/oci_registry.cpp | 24 +++++++++++++++++++ 2 files changed, 63 insertions(+), 9 deletions(-) diff --git a/src/oci/client.cpp b/src/oci/client.cpp index ef679159..53982f18 100644 --- a/src/oci/client.cpp +++ b/src/oci/client.cpp @@ -283,11 +283,18 @@ util::expected client::get_blob_to_file( if (!token) { return util::unexpected{token.error()}; } + + // stream to a temporary name and rename into place only after the + // transfer and digest verification succeed: a failed or interrupted + // download must never leave a file at the final path, where a later + // caller would mistake it for a complete blob. + const std::filesystem::path partial{file.string() + ".partial"}; + util::curl::request req; req.url = registry_url_ + impl::blob_path(repository_, d.string()); req.bearer_token = *token; req.follow_redirects = true; - req.download_file = file; + req.download_file = partial; req.on_download_progress = std::move(progress); req.should_abort = std::move(should_abort); @@ -305,28 +312,51 @@ util::expected client::get_blob_to_file( }; } + // every error path must remove the partial download before surfacing + // the error. + auto fail = [&partial](std::string msg) { + spdlog::debug("oci::get_blob_to_file removing partial download {}", + partial.string()); + std::error_code ec; + std::filesystem::remove(partial, ec); + if (ec) { + spdlog::warn("oci::get_blob_to_file unable to remove {}: {}", + partial.string(), ec.message()); + } + return util::unexpected{std::move(msg)}; + }; + + spdlog::debug("oci::get_blob_to_file downloading {} to {}", req.url, + partial.string()); auto resp = util::curl::perform(req); if (!resp) { - return util::unexpected{resp.error().message}; + return fail(resp.error().message); } if (resp->status != 200) { - return util::unexpected{ - fmt::format("failed to fetch blob {} (status {}): {}", d.string(), - resp->status, util::curl::http_message(resp->status))}; + return fail(fmt::format("failed to fetch blob {} (status {}): {}", + d.string(), resp->status, + util::curl::http_message(resp->status))); } if (verify) { const auto got = digest::from_sha256(util::sha256_final(hash)); if (got != d) { - std::error_code ec; - std::filesystem::remove(file, ec); - return util::unexpected{ + return fail( fmt::format("downloaded blob digest mismatch: expected {}, " "got {}", - d.string(), got.string())}; + d.string(), got.string())); } spdlog::debug("oci::get_blob_to_file verified {}", got.string()); } + + spdlog::debug("oci::get_blob_to_file moving {} to {}", partial.string(), + file.string()); + std::error_code ec; + std::filesystem::rename(partial, file, ec); + if (ec) { + return fail(fmt::format("unable to move downloaded blob {} to {}: {}", + partial.string(), file.string(), ec.message())); + } return {}; } diff --git a/test/unit/oci_registry.cpp b/test/unit/oci_registry.cpp index 4c88a33d..f9fe2b64 100644 --- a/test/unit/oci_registry.cpp +++ b/test/unit/oci_registry.cpp @@ -173,6 +173,30 @@ TEST_CASE("oci registry push/pull round-trip", "[registry]") { REQUIRE(read_file(store / "store.squashfs") == payload); } +TEST_CASE("oci registry failed blob download leaves no file", "[registry]") { + const auto base = registry_base(); + if (base.empty()) { + SKIP("no zot binary available for the registry tests"); + } + + auto c = oci::client::create(base, "test/app/1.0"); + REQUIRE(c.has_value()); + + // a valid digest that no blob in the registry has: the download must + // fail, and must not leave anything at the destination path (or a + // .partial next to it) that a later pull would mistake for a complete + // blob. + const auto missing = + oci::digest::from_sha256(util::sha256_string("no-such-blob")); + auto store = util::make_temp_dir().value(); + const auto dest = store / "store.squashfs"; + + auto pulled = c->get_blob_to_file(missing, dest); + REQUIRE(!pulled.has_value()); + REQUIRE(!std::filesystem::exists(dest)); + REQUIRE(!std::filesystem::exists(store / "store.squashfs.partial")); +} + TEST_CASE("oci registry attach + referrers + pull_meta", "[registry]") { const auto base = registry_base(); if (base.empty()) { From f8c307b6f4e77f06a253ca7bc020df387a1e7c92 Mon Sep 17 00:00:00 2001 From: bcumming Date: Fri, 10 Jul 2026 14:48:38 +0200 Subject: [PATCH 22/29] fix: support registries that do not provide referrers API --- meson.build | 1 + src/oci/auth.cpp | 51 +--------- src/oci/client.cpp | 187 +++++++++++++------------------------ src/oci/client.h | 10 +- src/oci/push.cpp | 9 +- src/oci/util.cpp | 149 +++++++++++++++++++++++++++++ src/oci/util.h | 68 ++++++++++++++ test/unit/oci_auth.cpp | 17 +--- test/unit/oci_client.cpp | 17 +--- test/unit/oci_registry.cpp | 12 +++ 10 files changed, 317 insertions(+), 204 deletions(-) create mode 100644 src/oci/util.cpp create mode 100644 src/oci/util.h diff --git a/meson.build b/meson.build index 0b86930d..91c51e66 100644 --- a/meson.build +++ b/meson.build @@ -68,6 +68,7 @@ lib_src = [ 'src/oci/pull.cpp', 'src/oci/push.cpp', 'src/oci/reference.cpp', + 'src/oci/util.cpp', 'src/uenv/parse.cpp', 'src/uenv/print.cpp', 'src/uenv/repository.cpp', diff --git a/src/oci/auth.cpp b/src/oci/auth.cpp index 2dfd16e8..3c3debb3 100644 --- a/src/oci/auth.cpp +++ b/src/oci/auth.cpp @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -21,52 +22,6 @@ namespace oci { -// --- pure helpers (no network; unit-tested) ----------------------------- -// Kept out of the public auth.h interface; tests redeclare these prototypes -// inside namespace oci::impl (see test/unit/oci_auth.cpp). The bearer-challenge -// parser lives in src/oci/parse.cpp (oci::parse_bearer_challenge). -namespace impl { - -std::string token_url(const bearer_challenge& challenge, - const std::vector& scopes) { - std::string url = challenge.realm; - char sep = url.find('?') == std::string::npos ? '?' : '&'; - if (!challenge.service.empty()) { - url += sep; - url += "service="; - url += challenge.service; - sep = '&'; - } - for (const auto& scope : scopes) { - url += sep; - url += "scope="; - url += scope; - sep = '&'; - } - return url; -} - -std::optional parse_token_response(std::string_view body) { - auto j = nlohmann::json::parse(body, nullptr, /*allow_exceptions=*/false); - if (j.is_discarded() || !j.is_object()) { - return std::nullopt; - } - if (auto it = j.find("token"); it != j.end() && it->is_string()) { - return it->get(); - } - if (auto it = j.find("access_token"); it != j.end() && it->is_string()) { - return it->get(); - } - return std::nullopt; -} - -std::string repository_scope(std::string_view repository, - std::string_view actions) { - return fmt::format("repository:{}:{}", repository, actions); -} - -} // namespace impl - util::expected, std::string> discover_challenge(const std::string& registry_url) { // normalise: drop any trailing '/' before appending the v2 base path. @@ -113,7 +68,7 @@ fetch_token(const bearer_challenge& challenge, const std::vector& scopes, const std::optional& creds) { util::curl::request req; - req.url = impl::token_url(challenge, scopes); + req.url = detail::token_url(challenge, scopes); if (creds) { req.username = creds->username; req.password = creds->password; @@ -130,7 +85,7 @@ fetch_token(const bearer_challenge& challenge, fmt::format("token request to {} failed with status {}: {}", req.url, resp->status, resp->body)}; } - auto token = impl::parse_token_response(resp->body); + auto token = detail::parse_token_response(resp->body); if (!token) { return util::unexpected{ "token endpoint response did not contain a token"}; diff --git a/src/oci/client.cpp b/src/oci/client.cpp index 53982f18..f1816274 100644 --- a/src/oci/client.cpp +++ b/src/oci/client.cpp @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -20,111 +21,6 @@ namespace oci { -// pure helpers (no network; unit-tested). Kept out of the public client.h -// interface; tests redeclare these prototypes inside namespace oci::impl (see -// test/unit/oci_client.cpp). -namespace impl { - -// defined in auth.cpp (also part of oci::impl). -std::string repository_scope(std::string_view repository, - std::string_view actions); - -std::string blob_path(std::string_view repository, std::string_view digest) { - return fmt::format("/v2/{}/blobs/{}", repository, digest); -} - -std::string manifest_path(std::string_view repository, - std::string_view reference) { - return fmt::format("/v2/{}/manifests/{}", repository, reference); -} - -std::string uploads_path(std::string_view repository) { - return fmt::format("/v2/{}/blobs/uploads/", repository); -} - -std::string tags_path(std::string_view repository) { - return fmt::format("/v2/{}/tags/list", repository); -} - -std::string referrers_path(std::string_view repository, - std::string_view digest) { - return fmt::format("/v2/{}/referrers/{}", repository, digest); -} - -std::string resolve_upload_url(std::string_view registry_url, - std::string_view location, - std::string_view digest) { - std::string url; - // the Location may be absolute (https://...) or registry-relative - // (/v2/...). - if (location.rfind("http://", 0) == 0 || - location.rfind("https://", 0) == 0) { - url = std::string{location}; - } else { - std::string base{registry_url}; - while (!base.empty() && base.back() == '/') { - base.pop_back(); - } - if (!location.empty() && location.front() != '/') { - base.push_back('/'); - } - url = base + std::string{location}; - } - // append the digest query parameter the monolithic PUT requires. - url += (url.find('?') == std::string::npos) ? '?' : '&'; - url += "digest="; - url += digest; - return url; -} - -std::optional> parse_tags_list(std::string_view body) { - auto j = nlohmann::json::parse(body, nullptr, /*allow_exceptions=*/false); - if (j.is_discarded() || !j.is_object()) { - return std::nullopt; - } - std::vector tags; - if (auto it = j.find("tags"); it != j.end() && it->is_array()) { - for (const auto& t : *it) { - if (t.is_string()) { - tags.push_back(t.get()); - } - } - } - return tags; -} - -std::optional> parse_referrers(std::string_view body) { - auto j = nlohmann::json::parse(body, nullptr, /*allow_exceptions=*/false); - if (j.is_discarded() || !j.is_object()) { - return std::nullopt; - } - std::vector out; - auto it = j.find("manifests"); - if (it == j.end() || !it->is_array()) { - return out; - } - for (const auto& m : *it) { - if (!m.is_object()) { - continue; - } - // a descriptor must carry a valid digest; skip malformed entries. - auto dg = digest::parse(m.value("digest", std::string{})); - if (!dg) { - continue; - } - descriptor d{.media_type = m.value("mediaType", std::string{}), - .digest = *dg, - .size = m.value("size", std::size_t{0})}; - if (auto a = m.find("artifactType"); a != m.end() && a->is_string()) { - d.artifact_type = a->get(); - } - out.push_back(std::move(d)); - } - return out; -} - -} // namespace impl - // --- registry addressing helpers (pure) --------------------------------- util::expected @@ -201,10 +97,10 @@ client::token_for(bool write) { } std::vector scopes; scopes.push_back( - impl::repository_scope(repository_, write ? "pull,push" : "pull")); + detail::repository_scope(repository_, write ? "pull,push" : "pull")); // cross-repo blob mounts need pull scope on the source repository too. for (const auto& r : extra_pull_scopes_) { - scopes.push_back(impl::repository_scope(r, "pull")); + scopes.push_back(detail::repository_scope(r, "pull")); } auto token = fetch_token(*challenge_, scopes, creds_); if (!token) { @@ -224,7 +120,7 @@ util::expected client::blob_exists(const digest& d) { return util::unexpected{token.error()}; } util::curl::request req; - req.url = registry_url_ + impl::blob_path(repository_, d.string()); + req.url = registry_url_ + detail::blob_path(repository_, d.string()); req.method = util::curl::http_method::head; req.bearer_token = *token; @@ -248,7 +144,7 @@ util::expected client::get_blob(const digest& d) { return util::unexpected{token.error()}; } util::curl::request req; - req.url = registry_url_ + impl::blob_path(repository_, d.string()); + req.url = registry_url_ + detail::blob_path(repository_, d.string()); req.bearer_token = *token; // registries 307-redirect blob downloads to backing storage. req.follow_redirects = true; @@ -291,7 +187,7 @@ util::expected client::get_blob_to_file( const std::filesystem::path partial{file.string() + ".partial"}; util::curl::request req; - req.url = registry_url_ + impl::blob_path(repository_, d.string()); + req.url = registry_url_ + detail::blob_path(repository_, d.string()); req.bearer_token = *token; req.follow_redirects = true; req.download_file = partial; @@ -367,7 +263,7 @@ client::get_manifest(const reference& ref) { return util::unexpected{token.error()}; } util::curl::request req; - req.url = registry_url_ + impl::manifest_path(repository_, ref.string()); + req.url = registry_url_ + detail::manifest_path(repository_, ref.string()); req.bearer_token = *token; req.header_lines = {fmt::format("Accept: {}", media_type_manifest), fmt::format("Accept: {}", media_type_index)}; @@ -398,7 +294,7 @@ util::expected, std::string> client::list_tags() { return util::unexpected{token.error()}; } util::curl::request req; - req.url = registry_url_ + impl::tags_path(repository_); + req.url = registry_url_ + detail::tags_path(repository_); req.bearer_token = *token; auto resp = util::curl::perform(req); @@ -409,7 +305,7 @@ util::expected, std::string> client::list_tags() { return util::unexpected{ fmt::format("failed to list tags (status {})", resp->status)}; } - auto tags = impl::parse_tags_list(resp->body); + auto tags = detail::parse_tags_list(resp->body); if (!tags) { return util::unexpected{"could not parse tags/list response"}; } @@ -423,25 +319,76 @@ client::referrers(const digest& d) { return util::unexpected{token.error()}; } util::curl::request req; - req.url = registry_url_ + impl::referrers_path(repository_, d.string()); + req.url = registry_url_ + detail::referrers_path(repository_, d.string()); req.bearer_token = *token; auto resp = util::curl::perform(req); if (!resp) { return util::unexpected{resp.error().message}; } + if (resp->status == 404) { + // per the OCI 1.1 distribution spec, a 404 means the registry does + // not implement the Referrers API (implementing registries return 200 + // with an empty index, even for a subject with no referrers). fall + // back to the referrers tag schema that the push side maintains. + spdlog::debug("oci::referrers {}: Referrers API unavailable (404), " + "falling back to the referrers tag schema", + d.string()); + return referrers_from_tag(d); + } if (resp->status != 200) { return util::unexpected{fmt::format( "failed to fetch referrers of {} (status {}): {}", d.string(), resp->status, util::curl::http_message(resp->status))}; } - auto refs = impl::parse_referrers(resp->body); + auto refs = detail::parse_referrers(resp->body); if (!refs) { return util::unexpected{"could not parse referrers response"}; } return *refs; } +util::expected, std::string> +client::referrers_from_tag(const digest& d) { + auto token = token_for(false); + if (!token) { + return util::unexpected{token.error()}; + } + // the tag schema names an OCI image index - in the same + // repository as the subject (see maintain_referrers_tag in push.cpp). + const auto tag = d.algorithm() + "-" + d.hex(); + util::curl::request req; + req.url = registry_url_ + detail::manifest_path(repository_, tag); + req.bearer_token = *token; + req.header_lines = {fmt::format("Accept: {}", media_type_index), + fmt::format("Accept: {}", media_type_manifest)}; + + spdlog::debug("oci::referrers_from_tag fetching {}", req.url); + auto resp = util::curl::perform(req); + if (!resp) { + return util::unexpected{resp.error().message}; + } + if (resp->status == 404) { + // the tag was never created: the subject has no referrers. + spdlog::debug("oci::referrers_from_tag {}: tag absent, no referrers", + tag); + return std::vector{}; + } + if (resp->status != 200) { + return util::unexpected{ + fmt::format("failed to fetch referrers tag {} (status {}): {}", tag, + resp->status, util::curl::http_message(resp->status))}; + } + auto refs = detail::parse_referrers(resp->body); + if (!refs) { + return util::unexpected{ + fmt::format("could not parse referrers tag index {}", tag)}; + } + spdlog::debug("oci::referrers_from_tag {}: {} referrer(s)", tag, + refs->size()); + return *refs; +} + namespace { // drive the monolithic upload handshake: POST an upload session, then PUT the @@ -472,7 +419,7 @@ do_put_blob(const std::string& registry_url, const std::string& uploads, // 2. PUT the payload to ?digest= util::curl::request put; - put.url = impl::resolve_upload_url(registry_url, *location, digest); + put.url = detail::resolve_upload_url(registry_url, *location, digest); put.method = util::curl::http_method::put; put.bearer_token = token; put.header_lines = { @@ -503,7 +450,7 @@ client::mount_blob(const digest& d, const std::string& from_repository) { return util::unexpected{token.error()}; } util::curl::request post; - post.url = registry_url_ + impl::uploads_path(repository_) + + post.url = registry_url_ + detail::uploads_path(repository_) + "?mount=" + d.string() + "&from=" + from_repository; post.method = util::curl::http_method::post; post.bearer_token = *token; @@ -540,7 +487,7 @@ client::put_blob(const digest& d, const std::filesystem::path& file, if (!token) { return util::unexpected{token.error()}; } - return do_put_blob(registry_url_, impl::uploads_path(repository_), *token, + return do_put_blob(registry_url_, detail::uploads_path(repository_), *token, d.string(), [&file, &progress](util::curl::request& r) { r.upload_file = file; if (progress) { @@ -558,7 +505,7 @@ client::put_blob_bytes(const digest& d, const std::string& data) { if (!token) { return util::unexpected{token.error()}; } - return do_put_blob(registry_url_, impl::uploads_path(repository_), *token, + return do_put_blob(registry_url_, detail::uploads_path(repository_), *token, d.string(), [&data](util::curl::request& r) { r.body = data; }); } @@ -571,7 +518,7 @@ client::put_manifest(const reference& ref, const std::string& body, return util::unexpected{token.error()}; } util::curl::request req; - req.url = registry_url_ + impl::manifest_path(repository_, ref.string()); + req.url = registry_url_ + detail::manifest_path(repository_, ref.string()); req.method = util::curl::http_method::put; req.bearer_token = *token; req.body = body; diff --git a/src/oci/client.h b/src/oci/client.h index ac6fdd24..9752e9c2 100644 --- a/src/oci/client.h +++ b/src/oci/client.h @@ -106,7 +106,10 @@ class client { // list the repository's tags. util::expected, std::string> list_tags(); - // list artifacts that refer to `d` (replaces `oras discover`). + // list artifacts that refer to `d` (replaces `oras discover`). registries + // that do not implement the OCI 1.1 Referrers API (404) are handled by + // falling back to the referrers tag schema (the - tag that the + // push side maintains); an absent tag means "no referrers". util::expected, std::string> referrers(const digest& d); @@ -156,6 +159,11 @@ class client { util::expected, std::string> token_for(bool write); + // read the referrers of `d` from the referrers tag schema index + // (-): the fallback for registries without the Referrers API. + util::expected, std::string> + referrers_from_tag(const digest& d); + std::string registry_url_; // normalised, no trailing '/' std::string repository_; std::optional creds_; diff --git a/src/oci/push.cpp b/src/oci/push.cpp index 1222b3cd..d44f3188 100644 --- a/src/oci/push.cpp +++ b/src/oci/push.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -21,12 +22,6 @@ namespace oci { -// defined in client.cpp (part of oci::impl, kept out of the public interface). -// parses the "manifests" array of an OCI image index into descriptors. -namespace impl { -std::optional> parse_referrers(std::string_view body); -} - namespace fs = std::filesystem; namespace { @@ -221,7 +216,7 @@ void maintain_referrers_tag(client& c, const digest& subject, // as an empty index. std::vector manifests; if (auto existing = c.get_manifest(tag)) { - if (auto refs = impl::parse_referrers(existing->body)) { + if (auto refs = detail::parse_referrers(existing->body)) { manifests = std::move(*refs); } } diff --git a/src/oci/util.cpp b/src/oci/util.cpp new file mode 100644 index 00000000..92e8b203 --- /dev/null +++ b/src/oci/util.cpp @@ -0,0 +1,149 @@ +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +namespace oci { +namespace detail { + +std::string blob_path(std::string_view repository, std::string_view digest) { + return fmt::format("/v2/{}/blobs/{}", repository, digest); +} + +std::string manifest_path(std::string_view repository, + std::string_view reference) { + return fmt::format("/v2/{}/manifests/{}", repository, reference); +} + +std::string uploads_path(std::string_view repository) { + return fmt::format("/v2/{}/blobs/uploads/", repository); +} + +std::string tags_path(std::string_view repository) { + return fmt::format("/v2/{}/tags/list", repository); +} + +std::string referrers_path(std::string_view repository, + std::string_view digest) { + return fmt::format("/v2/{}/referrers/{}", repository, digest); +} + +std::string resolve_upload_url(std::string_view registry_url, + std::string_view location, + std::string_view digest) { + std::string url; + // the Location may be absolute (https://...) or registry-relative + // (/v2/...). + if (location.rfind("http://", 0) == 0 || + location.rfind("https://", 0) == 0) { + url = std::string{location}; + } else { + std::string base{registry_url}; + while (!base.empty() && base.back() == '/') { + base.pop_back(); + } + if (!location.empty() && location.front() != '/') { + base.push_back('/'); + } + url = base + std::string{location}; + } + // append the digest query parameter the monolithic PUT requires. + url += (url.find('?') == std::string::npos) ? '?' : '&'; + url += "digest="; + url += digest; + return url; +} + +std::optional> parse_tags_list(std::string_view body) { + auto j = nlohmann::json::parse(body, nullptr, /*allow_exceptions=*/false); + if (j.is_discarded() || !j.is_object()) { + return std::nullopt; + } + std::vector tags; + if (auto it = j.find("tags"); it != j.end() && it->is_array()) { + for (const auto& t : *it) { + if (t.is_string()) { + tags.push_back(t.get()); + } + } + } + return tags; +} + +std::optional> parse_referrers(std::string_view body) { + auto j = nlohmann::json::parse(body, nullptr, /*allow_exceptions=*/false); + if (j.is_discarded() || !j.is_object()) { + return std::nullopt; + } + std::vector out; + auto it = j.find("manifests"); + if (it == j.end() || !it->is_array()) { + return out; + } + for (const auto& m : *it) { + if (!m.is_object()) { + continue; + } + // a descriptor must carry a valid digest; skip malformed entries. + auto dg = digest::parse(m.value("digest", std::string{})); + if (!dg) { + continue; + } + descriptor d{.media_type = m.value("mediaType", std::string{}), + .digest = *dg, + .size = m.value("size", std::size_t{0})}; + if (auto a = m.find("artifactType"); a != m.end() && a->is_string()) { + d.artifact_type = a->get(); + } + out.push_back(std::move(d)); + } + return out; +} + +std::string token_url(const bearer_challenge& challenge, + const std::vector& scopes) { + std::string url = challenge.realm; + char sep = url.find('?') == std::string::npos ? '?' : '&'; + if (!challenge.service.empty()) { + url += sep; + url += "service="; + url += challenge.service; + sep = '&'; + } + for (const auto& scope : scopes) { + url += sep; + url += "scope="; + url += scope; + sep = '&'; + } + return url; +} + +std::optional parse_token_response(std::string_view body) { + auto j = nlohmann::json::parse(body, nullptr, /*allow_exceptions=*/false); + if (j.is_discarded() || !j.is_object()) { + return std::nullopt; + } + if (auto it = j.find("token"); it != j.end() && it->is_string()) { + return it->get(); + } + if (auto it = j.find("access_token"); it != j.end() && it->is_string()) { + return it->get(); + } + return std::nullopt; +} + +std::string repository_scope(std::string_view repository, + std::string_view actions) { + return fmt::format("repository:{}:{}", repository, actions); +} + +} // namespace detail +} // namespace oci diff --git a/src/oci/util.h b/src/oci/util.h new file mode 100644 index 00000000..01a9ce30 --- /dev/null +++ b/src/oci/util.h @@ -0,0 +1,68 @@ +#pragma once + +// Internal helpers shared between the oci translation units and their unit +// tests. These are pure functions (no network, no state): URL/path builders +// and response-body parsers. They are deliberately kept out of the public +// client.h/auth.h interfaces — include this header only from src/oci/*.cpp +// and test/unit/oci_*.cpp. + +#include +#include +#include +#include + +#include +#include + +namespace oci { +namespace detail { + +// --- registry URL/path builders ------------------------------------------- + +std::string blob_path(std::string_view repository, std::string_view digest); + +std::string manifest_path(std::string_view repository, + std::string_view reference); + +std::string uploads_path(std::string_view repository); + +std::string tags_path(std::string_view repository); + +std::string referrers_path(std::string_view repository, + std::string_view digest); + +// resolve the Location header of an upload session (absolute or +// registry-relative) into the URL for the monolithic PUT, appending the +// ?digest= query parameter. +std::string resolve_upload_url(std::string_view registry_url, + std::string_view location, + std::string_view digest); + +// --- response-body parsers ------------------------------------------------- + +// parse a /v2//tags/list response body. +std::optional> parse_tags_list(std::string_view body); + +// parse the "manifests" array of an OCI image index (a Referrers API +// response, or a referrers tag-schema index) into descriptors. +std::optional> parse_referrers(std::string_view body); + +// --- token handshake helpers ----------------------------------------------- +// The bearer-challenge parser lives in src/oci/parse.cpp +// (oci::parse_bearer_challenge). + +// build the token endpoint URL from a bearer challenge and the requested +// scopes. +std::string token_url(const bearer_challenge& challenge, + const std::vector& scopes); + +// extract the token from a token endpoint response body ("token" or +// "access_token"). +std::optional parse_token_response(std::string_view body); + +// build a "repository::" scope string. +std::string repository_scope(std::string_view repository, + std::string_view actions); + +} // namespace detail +} // namespace oci diff --git a/test/unit/oci_auth.cpp b/test/unit/oci_auth.cpp index 8cc4bb4e..bc6b3145 100644 --- a/test/unit/oci_auth.cpp +++ b/test/unit/oci_auth.cpp @@ -5,21 +5,12 @@ #include #include +#include #include -// the pure helpers are not part of the public auth.h interface; redeclare their -// prototypes here (they live in namespace oci::impl in auth.cpp). the -// bearer-challenge parser moved to src/oci/parse.cpp and is covered by -// test/unit/oci_parse.cpp. -namespace oci::impl { -std::string token_url(const bearer_challenge&, const std::vector&); -std::optional parse_token_response(std::string_view); -std::string repository_scope(std::string_view, std::string_view); -} // namespace oci::impl - -using oci::impl::parse_token_response; -using oci::impl::repository_scope; -using oci::impl::token_url; +using oci::detail::parse_token_response; +using oci::detail::repository_scope; +using oci::detail::token_url; TEST_CASE("oci token_url", "[oci][auth]") { oci::bearer_challenge c{.realm = "https://jfrog.svc.cscs.ch/v2/token", diff --git a/test/unit/oci_client.cpp b/test/unit/oci_client.cpp index 6d36c291..335b6ee9 100644 --- a/test/unit/oci_client.cpp +++ b/test/unit/oci_client.cpp @@ -4,24 +4,11 @@ #include #include +#include #include -// the pure helpers are not part of the public client.h interface; redeclare -// their prototypes here (they live in namespace oci::impl in client.cpp). -namespace oci::impl { -std::string blob_path(std::string_view, std::string_view); -std::string manifest_path(std::string_view, std::string_view); -std::string uploads_path(std::string_view); -std::string tags_path(std::string_view); -std::string referrers_path(std::string_view, std::string_view); -std::string resolve_upload_url(std::string_view, std::string_view, - std::string_view); -std::optional> parse_tags_list(std::string_view); -std::optional> parse_referrers(std::string_view); -} // namespace oci::impl - using namespace oci; -using namespace oci::impl; +using namespace oci::detail; TEST_CASE("oci path builders", "[oci][client]") { const std::string repo = "deploy/todi/gh200/app/1.0"; diff --git a/test/unit/oci_registry.cpp b/test/unit/oci_registry.cpp index f9fe2b64..c0d7ac72 100644 --- a/test/unit/oci_registry.cpp +++ b/test/unit/oci_registry.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -230,6 +231,17 @@ TEST_CASE("oci registry attach + referrers + pull_meta", "[registry]") { } REQUIRE(found_meta); + // the referrers tag schema (-) is the fallback used by + // client::referrers on registries without the Referrers API; attach must + // maintain it, and its index must list the same referrers as the API. + const auto fallback_tag = + oci::reference::tag(pushed->algorithm() + "-" + pushed->hex()); + auto tag_index = c->get_manifest(fallback_tag); + REQUIRE(tag_index.has_value()); + auto tag_refs = oci::detail::parse_referrers(tag_index->body); + REQUIRE(tag_refs.has_value()); + REQUIRE(*tag_refs == *refs); + auto store = util::make_temp_dir().value(); auto got = oci::pull_meta(*c, *pushed, store); REQUIRE(got.has_value()); From 80fbce2045fd87f851024a454df487820bb400d2 Mon Sep 17 00:00:00 2001 From: bcumming Date: Fri, 10 Jul 2026 15:28:11 +0200 Subject: [PATCH 23/29] dsitinguish between error types when getting manifests --- src/oci/client.cpp | 180 ++++++++++++++++++++----------------- src/oci/client.h | 51 ++++++++--- src/oci/pull.cpp | 8 +- src/oci/push.cpp | 51 +++++++---- test/unit/oci_registry.cpp | 22 +++++ 5 files changed, 198 insertions(+), 114 deletions(-) diff --git a/src/oci/client.cpp b/src/oci/client.cpp index f1816274..5f8a14f4 100644 --- a/src/oci/client.cpp +++ b/src/oci/client.cpp @@ -66,12 +66,12 @@ std::string repository_path(std::string_view prefix, std::string_view nspace, // --- client ------------------------------------------------------------- -util::expected +util::expected client::create(std::string registry_url, std::string repository, std::optional creds) { auto challenge = discover_challenge(registry_url); if (!challenge) { - return util::unexpected{challenge.error()}; + return util::unexpected{client_error{challenge.error()}}; } client c; @@ -114,10 +114,10 @@ void client::add_pull_scope(std::string repository) { extra_pull_scopes_.push_back(std::move(repository)); } -util::expected client::blob_exists(const digest& d) { +util::expected client::blob_exists(const digest& d) { auto token = token_for(false); if (!token) { - return util::unexpected{token.error()}; + return util::unexpected{client_error{token.error()}}; } util::curl::request req; req.url = registry_url_ + detail::blob_path(repository_, d.string()); @@ -126,7 +126,7 @@ util::expected client::blob_exists(const digest& d) { auto resp = util::curl::perform(req); if (!resp) { - return util::unexpected{resp.error().message}; + return util::unexpected{client_error{resp.error().message}}; } if (resp->status == 200) { return true; @@ -134,14 +134,16 @@ util::expected client::blob_exists(const digest& d) { if (resp->status == 404) { return false; } - return util::unexpected{fmt::format("unexpected status {} for HEAD {}", - resp->status, d.string())}; + return util::unexpected{ + client_error{fmt::format("unexpected status {} for HEAD {}", + resp->status, d.string()), + resp->status}}; } -util::expected client::get_blob(const digest& d) { +util::expected client::get_blob(const digest& d) { auto token = token_for(false); if (!token) { - return util::unexpected{token.error()}; + return util::unexpected{client_error{token.error()}}; } util::curl::request req; req.url = registry_url_ + detail::blob_path(repository_, d.string()); @@ -151,17 +153,18 @@ util::expected client::get_blob(const digest& d) { auto resp = util::curl::perform(req); if (!resp) { - return util::unexpected{resp.error().message}; + return util::unexpected{client_error{resp.error().message}}; } if (resp->status != 200) { - return util::unexpected{ + return util::unexpected{client_error{ fmt::format("failed to fetch blob {} (status {}): {}", d.string(), - resp->status, util::curl::http_message(resp->status))}; + resp->status, util::curl::http_message(resp->status)), + resp->status}}; } return resp->body; } -util::expected client::get_blob_to_file( +util::expected client::get_blob_to_file( const digest& d, const std::filesystem::path& file, std::function progress, std::function should_abort) { @@ -170,14 +173,14 @@ util::expected client::get_blob_to_file( const auto parent = file.parent_path(); if (!parent.empty() && util::file_access_level(parent) != util::file_level::readwrite) { - return util::unexpected{fmt::format( + return util::unexpected{client_error{fmt::format( "cannot write blob to {}: {} is not a writable directory", - file.string(), parent.string())}; + file.string(), parent.string())}}; } auto token = token_for(false); if (!token) { - return util::unexpected{token.error()}; + return util::unexpected{client_error{token.error()}}; } // stream to a temporary name and rename into place only after the @@ -210,7 +213,7 @@ util::expected client::get_blob_to_file( // every error path must remove the partial download before surfacing // the error. - auto fail = [&partial](std::string msg) { + auto fail = [&partial](client_error err) { spdlog::debug("oci::get_blob_to_file removing partial download {}", partial.string()); std::error_code ec; @@ -219,28 +222,29 @@ util::expected client::get_blob_to_file( spdlog::warn("oci::get_blob_to_file unable to remove {}: {}", partial.string(), ec.message()); } - return util::unexpected{std::move(msg)}; + return util::unexpected{std::move(err)}; }; spdlog::debug("oci::get_blob_to_file downloading {} to {}", req.url, partial.string()); auto resp = util::curl::perform(req); if (!resp) { - return fail(resp.error().message); + return fail({resp.error().message}); } if (resp->status != 200) { - return fail(fmt::format("failed to fetch blob {} (status {}): {}", - d.string(), resp->status, - util::curl::http_message(resp->status))); + return fail( + {fmt::format("failed to fetch blob {} (status {}): {}", d.string(), + resp->status, util::curl::http_message(resp->status)), + resp->status}); } if (verify) { const auto got = digest::from_sha256(util::sha256_final(hash)); if (got != d) { return fail( - fmt::format("downloaded blob digest mismatch: expected {}, " - "got {}", - d.string(), got.string())); + {fmt::format("downloaded blob digest mismatch: expected {}, " + "got {}", + d.string(), got.string())}); } spdlog::debug("oci::get_blob_to_file verified {}", got.string()); } @@ -250,17 +254,18 @@ util::expected client::get_blob_to_file( std::error_code ec; std::filesystem::rename(partial, file, ec); if (ec) { - return fail(fmt::format("unable to move downloaded blob {} to {}: {}", - partial.string(), file.string(), ec.message())); + return fail( + {fmt::format("unable to move downloaded blob {} to {}: {}", + partial.string(), file.string(), ec.message())}); } return {}; } -util::expected +util::expected client::get_manifest(const reference& ref) { auto token = token_for(false); if (!token) { - return util::unexpected{token.error()}; + return util::unexpected{client_error{token.error()}}; } util::curl::request req; req.url = registry_url_ + detail::manifest_path(repository_, ref.string()); @@ -270,12 +275,14 @@ client::get_manifest(const reference& ref) { auto resp = util::curl::perform(req); if (!resp) { - return util::unexpected{resp.error().message}; + return util::unexpected{client_error{resp.error().message}}; } if (resp->status != 200) { - return util::unexpected{fmt::format( - "failed to fetch manifest {} (status {}): {}", ref.string(), - resp->status, util::curl::http_message(resp->status))}; + return util::unexpected{client_error{ + fmt::format("failed to fetch manifest {} (status {}): {}", + ref.string(), resp->status, + util::curl::http_message(resp->status)), + resp->status}}; } manifest_response m; m.body = std::move(resp->body); @@ -288,10 +295,10 @@ client::get_manifest(const reference& ref) { return m; } -util::expected, std::string> client::list_tags() { +util::expected, client_error> client::list_tags() { auto token = token_for(false); if (!token) { - return util::unexpected{token.error()}; + return util::unexpected{client_error{token.error()}}; } util::curl::request req; req.url = registry_url_ + detail::tags_path(repository_); @@ -299,24 +306,26 @@ util::expected, std::string> client::list_tags() { auto resp = util::curl::perform(req); if (!resp) { - return util::unexpected{resp.error().message}; + return util::unexpected{client_error{resp.error().message}}; } if (resp->status != 200) { - return util::unexpected{ - fmt::format("failed to list tags (status {})", resp->status)}; + return util::unexpected{client_error{ + fmt::format("failed to list tags (status {})", resp->status), + resp->status}}; } auto tags = detail::parse_tags_list(resp->body); if (!tags) { - return util::unexpected{"could not parse tags/list response"}; + return util::unexpected{ + client_error{"could not parse tags/list response"}}; } return *tags; } -util::expected, std::string> +util::expected, client_error> client::referrers(const digest& d) { auto token = token_for(false); if (!token) { - return util::unexpected{token.error()}; + return util::unexpected{client_error{token.error()}}; } util::curl::request req; req.url = registry_url_ + detail::referrers_path(repository_, d.string()); @@ -324,7 +333,7 @@ client::referrers(const digest& d) { auto resp = util::curl::perform(req); if (!resp) { - return util::unexpected{resp.error().message}; + return util::unexpected{client_error{resp.error().message}}; } if (resp->status == 404) { // per the OCI 1.1 distribution spec, a 404 means the registry does @@ -337,22 +346,25 @@ client::referrers(const digest& d) { return referrers_from_tag(d); } if (resp->status != 200) { - return util::unexpected{fmt::format( - "failed to fetch referrers of {} (status {}): {}", d.string(), - resp->status, util::curl::http_message(resp->status))}; + return util::unexpected{client_error{ + fmt::format("failed to fetch referrers of {} (status {}): {}", + d.string(), resp->status, + util::curl::http_message(resp->status)), + resp->status}}; } auto refs = detail::parse_referrers(resp->body); if (!refs) { - return util::unexpected{"could not parse referrers response"}; + return util::unexpected{ + client_error{"could not parse referrers response"}}; } return *refs; } -util::expected, std::string> +util::expected, client_error> client::referrers_from_tag(const digest& d) { auto token = token_for(false); if (!token) { - return util::unexpected{token.error()}; + return util::unexpected{client_error{token.error()}}; } // the tag schema names an OCI image index - in the same // repository as the subject (see maintain_referrers_tag in push.cpp). @@ -366,7 +378,7 @@ client::referrers_from_tag(const digest& d) { spdlog::debug("oci::referrers_from_tag fetching {}", req.url); auto resp = util::curl::perform(req); if (!resp) { - return util::unexpected{resp.error().message}; + return util::unexpected{client_error{resp.error().message}}; } if (resp->status == 404) { // the tag was never created: the subject has no referrers. @@ -375,14 +387,15 @@ client::referrers_from_tag(const digest& d) { return std::vector{}; } if (resp->status != 200) { - return util::unexpected{ + return util::unexpected{client_error{ fmt::format("failed to fetch referrers tag {} (status {}): {}", tag, - resp->status, util::curl::http_message(resp->status))}; + resp->status, util::curl::http_message(resp->status)), + resp->status}}; } auto refs = detail::parse_referrers(resp->body); if (!refs) { - return util::unexpected{ - fmt::format("could not parse referrers tag index {}", tag)}; + return util::unexpected{client_error{ + fmt::format("could not parse referrers tag index {}", tag)}}; } spdlog::debug("oci::referrers_from_tag {}: {} referrer(s)", tag, refs->size()); @@ -393,7 +406,7 @@ namespace { // drive the monolithic upload handshake: POST an upload session, then PUT the // payload (carried by `body_setup`) to the returned Location with ?digest=. -util::expected +util::expected do_put_blob(const std::string& registry_url, const std::string& uploads, const std::optional& token, const std::string& digest, const std::function& body_setup) { @@ -404,17 +417,19 @@ do_put_blob(const std::string& registry_url, const std::string& uploads, post.bearer_token = token; auto opened = util::curl::perform(post); if (!opened) { - return util::unexpected{opened.error().message}; + return util::unexpected{client_error{opened.error().message}}; } if (opened->status != 202) { - return util::unexpected{fmt::format( - "failed to open upload session (status {}): {} {}", opened->status, - util::curl::http_message(opened->status), opened->body)}; + return util::unexpected{client_error{ + fmt::format("failed to open upload session (status {}): {} {}", + opened->status, + util::curl::http_message(opened->status), opened->body), + opened->status}}; } auto location = opened->headers.get("location"); if (!location) { return util::unexpected{ - "upload session response had no Location header"}; + client_error{"upload session response had no Location header"}}; } // 2. PUT the payload to ?digest= @@ -428,26 +443,28 @@ do_put_blob(const std::string& registry_url, const std::string& uploads, auto done = util::curl::perform(put); if (!done) { - return util::unexpected{done.error().message}; + return util::unexpected{client_error{done.error().message}}; } if (done->status != 201) { - return util::unexpected{fmt::format( - "failed to upload blob {} (status {}): {} {}", digest, done->status, - util::curl::http_message(done->status), done->body)}; + return util::unexpected{client_error{ + fmt::format("failed to upload blob {} (status {}): {} {}", digest, + done->status, util::curl::http_message(done->status), + done->body), + done->status}}; } return {}; } } // namespace -util::expected +util::expected client::mount_blob(const digest& d, const std::string& from_repository) { if (auto exists = blob_exists(d); exists && *exists) { return true; } auto token = token_for(true); if (!token) { - return util::unexpected{token.error()}; + return util::unexpected{client_error{token.error()}}; } util::curl::request post; post.url = registry_url_ + detail::uploads_path(repository_) + @@ -457,7 +474,7 @@ client::mount_blob(const digest& d, const std::string& from_repository) { auto resp = util::curl::perform(post); if (!resp) { - return util::unexpected{resp.error().message}; + return util::unexpected{client_error{resp.error().message}}; } // 201: the blob was mounted. 202: the registry declined and opened an // upload session instead (caller must copy the blob the slow way). @@ -467,17 +484,18 @@ client::mount_blob(const digest& d, const std::string& from_repository) { if (resp->status == 202) { return false; } - return util::unexpected{ + return util::unexpected{client_error{ fmt::format("failed to mount blob {} from {} (status {}): {}", - d.string(), from_repository, resp->status, resp->body)}; + d.string(), from_repository, resp->status, resp->body), + resp->status}}; } -util::expected +util::expected client::put_blob(const digest& d, const std::filesystem::path& file, std::function progress) { if (util::file_access_level(file) < util::file_level::readonly) { - return util::unexpected{fmt::format( - "cannot upload blob: {} is not a readable file", file.string())}; + return util::unexpected{client_error{fmt::format( + "cannot upload blob: {} is not a readable file", file.string())}}; } if (auto exists = blob_exists(d); exists && *exists) { spdlog::trace("oci::put_blob {} already present", d.string()); @@ -485,7 +503,7 @@ client::put_blob(const digest& d, const std::filesystem::path& file, } auto token = token_for(true); if (!token) { - return util::unexpected{token.error()}; + return util::unexpected{client_error{token.error()}}; } return do_put_blob(registry_url_, detail::uploads_path(repository_), *token, d.string(), [&file, &progress](util::curl::request& r) { @@ -496,26 +514,26 @@ client::put_blob(const digest& d, const std::filesystem::path& file, }); } -util::expected +util::expected client::put_blob_bytes(const digest& d, const std::string& data) { if (auto exists = blob_exists(d); exists && *exists) { return {}; } auto token = token_for(true); if (!token) { - return util::unexpected{token.error()}; + return util::unexpected{client_error{token.error()}}; } return do_put_blob(registry_url_, detail::uploads_path(repository_), *token, d.string(), [&data](util::curl::request& r) { r.body = data; }); } -util::expected +util::expected client::put_manifest(const reference& ref, const std::string& body, std::string_view media_type) { auto token = token_for(true); if (!token) { - return util::unexpected{token.error()}; + return util::unexpected{client_error{token.error()}}; } util::curl::request req; req.url = registry_url_ + detail::manifest_path(repository_, ref.string()); @@ -526,12 +544,14 @@ client::put_manifest(const reference& ref, const std::string& body, auto resp = util::curl::perform(req); if (!resp) { - return util::unexpected{resp.error().message}; + return util::unexpected{client_error{resp.error().message}}; } if (resp->status != 201) { - return util::unexpected{fmt::format( - "failed to put manifest {} (status {}): {} {}", ref.string(), - resp->status, util::curl::http_message(resp->status), resp->body)}; + return util::unexpected{client_error{ + fmt::format("failed to put manifest {} (status {}): {} {}", + ref.string(), resp->status, + util::curl::http_message(resp->status), resp->body), + resp->status}}; } return {}; } diff --git a/src/oci/client.h b/src/oci/client.h index 9752e9c2..4e19ef7b 100644 --- a/src/oci/client.h +++ b/src/oci/client.h @@ -70,6 +70,19 @@ struct manifest_response { std::string media_type; }; +// The error type of client registry operations: a human-readable message, +// plus the HTTP status code of the failed request when a response was +// received. `http_status` is nullopt for failures that never produced an +// HTTP response: transport errors (DNS, connect, TLS, dropped connection), +// token-fetch failures, and local errors (unwritable destination, digest +// mismatch). Callers that need to distinguish "not found" (often an expected +// outcome) from a transient failure branch on `http_status`; everything else +// just prints `message`. +struct client_error { + std::string message; + std::optional http_status = std::nullopt; +}; + // --- registry client ---------------------------------------------------- // A client bound to one repository on one registry. Authenticates lazily, @@ -80,37 +93,37 @@ class client { // Probe the registry, parse its auth challenge, and bind to `repository` // (e.g. "deploy/todi/gh200/app/1.0"). Supply credentials for push or // private pull; omit for anonymous pull. - static util::expected + static util::expected create(std::string registry_url, std::string repository, std::optional creds = std::nullopt); // does a blob exist? HEAD; 200 -> true, 404 -> false. - util::expected blob_exists(const digest& d); + util::expected blob_exists(const digest& d); // fetch a blob's bytes (follows the 307 redirect to backing storage). - util::expected get_blob(const digest& d); + util::expected get_blob(const digest& d); // stream a blob straight to a file, never holding it in memory (for the // multi-GB squashfs layer). follows the 307 redirect to backing storage. // an optional progress callback receives (bytes_downloaded, bytes_total); // an optional abort predicate, polled during transfer, cancels it. - util::expected get_blob_to_file( + util::expected get_blob_to_file( const digest& d, const std::filesystem::path& file, std::function progress = {}, std::function should_abort = {}); // fetch a manifest by tag or digest. - util::expected + util::expected get_manifest(const reference& ref); // list the repository's tags. - util::expected, std::string> list_tags(); + util::expected, client_error> list_tags(); // list artifacts that refer to `d` (replaces `oras discover`). registries // that do not implement the OCI 1.1 Referrers API (404) are handled by // falling back to the referrers tag schema (the - tag that the // push side maintains); an absent tag means "no referrers". - util::expected, std::string> + util::expected, client_error> referrers(const digest& d); // attempt a cross-repository blob mount from `from_repository` into this @@ -118,23 +131,23 @@ class client { // registry mounted the blob (201), false if it declined and wants a full // upload instead (202) — the caller should then fall back to copying. no-op // (returns true) if the blob already exists here. - util::expected + util::expected mount_blob(const digest& d, const std::string& from_repository); // upload a blob via the monolithic POST-then-PUT handshake. the file is // streamed from disk (not held in memory), so large squashfs layers are // fine. no-op if the blob already exists. an optional progress callback // receives (bytes_uploaded, bytes_total) during the PUT. - util::expected + util::expected put_blob(const digest& d, const std::filesystem::path& file, std::function progress = {}); // upload an in-memory blob (config blobs, small payloads). - util::expected put_blob_bytes(const digest& d, - const std::string& data); + util::expected put_blob_bytes(const digest& d, + const std::string& data); // PUT a manifest under `ref` (tag or digest). - util::expected + util::expected put_manifest(const reference& ref, const std::string& body, std::string_view media_type = media_type_manifest); @@ -161,7 +174,7 @@ class client { // read the referrers of `d` from the referrers tag schema index // (-): the fallback for registries without the Referrers API. - util::expected, std::string> + util::expected, client_error> referrers_from_tag(const digest& d); std::string registry_url_; // normalised, no trailing '/' @@ -177,3 +190,15 @@ class client { }; } // namespace oci + +#include +template <> class fmt::formatter { + public: + constexpr auto parse(format_parse_context& ctx) { + return ctx.end(); + } + template + auto format(oci::client_error const& e, FmtContext& ctx) const { + return fmt::format_to(ctx.out(), "{}", e.message); + } +}; diff --git a/src/oci/pull.cpp b/src/oci/pull.cpp index 9418660b..abe78306 100644 --- a/src/oci/pull.cpp +++ b/src/oci/pull.cpp @@ -89,7 +89,7 @@ pull_squashfs(client& c, const manifest& image, const fs::path& store, if (auto ok = c.get_blob_to_file(want, dest, std::move(progress), std::move(should_abort)); !ok) { - return util::unexpected{ok.error()}; + return util::unexpected{ok.error().message}; } return {}; } @@ -98,7 +98,7 @@ util::expected pull_meta(client& c, const digest& manifest_digest, const fs::path& store) { auto refs = c.referrers(manifest_digest); if (!refs) { - return util::unexpected{refs.error()}; + return util::unexpected{refs.error().message}; } // find the uenv/meta referrer. @@ -115,7 +115,7 @@ pull_meta(client& c, const digest& manifest_digest, const fs::path& store) { auto meta_manifest = c.get_manifest(reference::digest(meta->digest)); if (!meta_manifest) { - return util::unexpected{meta_manifest.error()}; + return util::unexpected{meta_manifest.error().message}; } auto parsed = parse_manifest(meta_manifest->body); if (!parsed) { @@ -135,7 +135,7 @@ pull_meta(client& c, const digest& manifest_digest, const fs::path& store) { spdlog::debug("oci::pull_meta layer {}", layer->digest.string()); auto blob = c.get_blob(layer->digest); if (!blob) { - return util::unexpected{blob.error()}; + return util::unexpected{blob.error().message}; } if (auto ok = util::ensure_directory(store); !ok) { diff --git a/src/oci/push.cpp b/src/oci/push.cpp index d44f3188..12f411af 100644 --- a/src/oci/push.cpp +++ b/src/oci/push.cpp @@ -50,7 +50,7 @@ digest digest_of_string(std::string_view s) { } // upload the canonical empty config blob (idempotent). -util::expected put_empty_config(client& c) { +util::expected put_empty_config(client& c) { return c.put_blob_bytes(empty_config_descriptor().digest, std::string{empty_config_body}); } @@ -179,7 +179,7 @@ util::expected copy_blob(client& src, client& dst, const digest& d) { auto mounted = dst.mount_blob(d, from_repo); if (!mounted) { - return util::unexpected{mounted.error()}; + return util::unexpected{mounted.error().message}; } if (*mounted) { spdlog::trace("copy_blob mounted {}", d.string()); @@ -194,10 +194,10 @@ util::expected copy_blob(client& src, client& dst, } const auto tmp = *work / "blob"; if (auto ok = src.get_blob_to_file(d, tmp); !ok) { - return util::unexpected{ok.error()}; + return util::unexpected{ok.error().message}; } if (auto ok = dst.put_blob(d, tmp); !ok) { - return util::unexpected{ok.error()}; + return util::unexpected{ok.error().message}; } std::error_code ec; fs::remove(tmp, ec); @@ -212,13 +212,30 @@ void maintain_referrers_tag(client& c, const digest& subject, const auto tag = reference::tag(subject.algorithm() + "-" + subject.hex()); // read the existing index (an OCI image index, i.e. a manifests[] list) so - // we merge rather than clobber prior referrers. A 404/parse-miss is treated - // as an empty index. + // we merge rather than clobber prior referrers. std::vector manifests; - if (auto existing = c.get_manifest(tag)) { + auto existing = c.get_manifest(tag); + if (existing) { if (auto refs = detail::parse_referrers(existing->body)) { manifests = std::move(*refs); + } else { + // 200 with an unparseable body: the index is corrupt and useless + // to every client, so rebuilding it loses nothing. + spdlog::warn("referrers tag {} holds an unparseable index; " + "rebuilding it", + tag.string()); } + } else if (existing.error().http_status == 404) { + // the tag has never been created: start a new index. + spdlog::debug("referrers tag {} not found, creating it", tag.string()); + } else { + // any other failure (transport, auth, 5xx) says nothing about the + // tag's contents: it may hold referrers that rebuilding from scratch + // would silently discard, so leave it unchanged. + spdlog::warn("unable to read referrers tag {} ({}); leaving it " + "unchanged", + tag.string(), existing.error().message); + return; } for (const auto& d : manifests) { if (d.digest == referrer.digest) { @@ -256,10 +273,10 @@ push_squashfs(client& c, const fs::path& squashfs, const reference& ref, // upload the squashfs blob (streamed) and the empty config. if (auto ok = c.put_blob(*layer_digest, squashfs, std::move(progress)); !ok) { - return util::unexpected{ok.error()}; + return util::unexpected{ok.error().message}; } if (auto ok = put_empty_config(c); !ok) { - return util::unexpected{ok.error()}; + return util::unexpected{ok.error().message}; } manifest m; @@ -273,7 +290,7 @@ push_squashfs(client& c, const fs::path& squashfs, const reference& ref, auto body = serialize_manifest(m); if (auto ok = c.put_manifest(ref, body); !ok) { - return util::unexpected{ok.error()}; + return util::unexpected{ok.error().message}; } // the canonical id is the digest of the manifest bytes we just PUT. @@ -327,11 +344,11 @@ util::expected attach(client& c, // upload the layer blob and empty config. if (auto ok = c.put_blob(packaged->layer.digest, packaged->blob); !ok) { cleanup(); - return util::unexpected{ok.error()}; + return util::unexpected{ok.error().message}; } cleanup(); if (auto ok = put_empty_config(c); !ok) { - return util::unexpected{ok.error()}; + return util::unexpected{ok.error().message}; } // build + push the referrer manifest. @@ -345,7 +362,7 @@ util::expected attach(client& c, const auto manifest_digest = digest_of_string(body); if (auto ok = c.put_manifest(reference::digest(manifest_digest), body); !ok) { - return util::unexpected{ok.error()}; + return util::unexpected{ok.error().message}; } descriptor referrer{.media_type = std::string{media_type_manifest}, @@ -366,11 +383,11 @@ copy_image(const std::string& registry_base, const std::string& src_repo, const std::string& dst_tag, std::optional creds) { auto src = client::create(registry_base, src_repo, creds); if (!src) { - return util::unexpected{src.error()}; + return util::unexpected{src.error().message}; } auto dst = client::create(registry_base, dst_repo, creds); if (!dst) { - return util::unexpected{dst.error()}; + return util::unexpected{dst.error().message}; } // dst tokens need pull scope on the source repo for cross-repo mounts. dst->add_pull_scope(src_repo); @@ -401,7 +418,7 @@ copy_image(const std::string& registry_base, const std::string& src_repo, if (auto ok = dst->put_manifest(reference::tag(dst_tag), mr->body, manifest_media); !ok) { - return util::unexpected{ok.error()}; + return util::unexpected{ok.error().message}; } // copy referrers (the --recursive part): meta and any other attachments. @@ -437,7 +454,7 @@ copy_image(const std::string& registry_base, const std::string& src_repo, if (auto ok = dst->put_manifest(reference::digest(r.digest), rm->body, rmedia); !ok) { - return util::unexpected{ok.error()}; + return util::unexpected{ok.error().message}; } } diff --git a/test/unit/oci_registry.cpp b/test/unit/oci_registry.cpp index c0d7ac72..c6be114a 100644 --- a/test/unit/oci_registry.cpp +++ b/test/unit/oci_registry.cpp @@ -242,6 +242,28 @@ TEST_CASE("oci registry attach + referrers + pull_meta", "[registry]") { REQUIRE(tag_refs.has_value()); REQUIRE(*tag_refs == *refs); + // attaching a second artifact must merge into the tag index, not clobber + // the entry the first attach wrote. + const auto extra = dir / "notes.json"; + write_file(extra, R"({"note":"x"})"); + auto att2 = + oci::attach(*c, oci::reference::digest(*pushed), "uenv/extra", extra); + REQUIRE(att2.has_value()); + + auto tag_index2 = c->get_manifest(fallback_tag); + REQUIRE(tag_index2.has_value()); + auto tag_refs2 = oci::detail::parse_referrers(tag_index2->body); + REQUIRE(tag_refs2.has_value()); + REQUIRE(tag_refs2->size() == 2); + bool tag_has_meta = false; + bool tag_has_extra = false; + for (const auto& d : *tag_refs2) { + tag_has_meta = tag_has_meta || d.artifact_type == "uenv/meta"; + tag_has_extra = tag_has_extra || d.artifact_type == "uenv/extra"; + } + REQUIRE(tag_has_meta); + REQUIRE(tag_has_extra); + auto store = util::make_temp_dir().value(); auto got = oci::pull_meta(*c, *pushed, store); REQUIRE(got.has_value()); From 6b451df5b076bb5142ddad24e2b0433bc317fd77 Mon Sep 17 00:00:00 2001 From: bcumming Date: Fri, 10 Jul 2026 15:39:36 +0200 Subject: [PATCH 24/29] bugfix: re-authenticate expired registry tokens, and stop image copy silently dropping attached metadata --- src/oci/auth.cpp | 8 +- src/oci/auth.h | 14 ++- src/oci/client.cpp | 203 +++++++++++++++++++------------------ src/oci/client.h | 43 +++++++- src/oci/push.cpp | 17 +++- src/oci/util.cpp | 17 +++- src/oci/util.h | 7 +- test/unit/oci_auth.cpp | 32 ++++-- test/unit/oci_registry.cpp | 28 +++++ 9 files changed, 240 insertions(+), 129 deletions(-) diff --git a/src/oci/auth.cpp b/src/oci/auth.cpp index 3c3debb3..0e027023 100644 --- a/src/oci/auth.cpp +++ b/src/oci/auth.cpp @@ -63,7 +63,7 @@ discover_challenge(const std::string& registry_url) { return *challenge; } -util::expected +util::expected fetch_token(const bearer_challenge& challenge, const std::vector& scopes, const std::optional& creds) { @@ -105,7 +105,11 @@ authenticate(const std::string& registry_url, if (!*challenge) { return std::string{}; } - return fetch_token(**challenge, scopes, creds); + auto token = fetch_token(**challenge, scopes, creds); + if (!token) { + return util::unexpected{token.error()}; + } + return token->token; } util::expected, std::string> diff --git a/src/oci/auth.h b/src/oci/auth.h index 565f30c5..9c391d4c 100644 --- a/src/oci/auth.h +++ b/src/oci/auth.h @@ -28,6 +28,16 @@ struct bearer_challenge { const bearer_challenge&) = default; }; +// A token issued by the registry's token endpoint: the raw bearer token (for +// `Authorization: Bearer `) plus its advertised lifetime in seconds, +// when the endpoint reported one (`expires_in` is optional in the token +// response; registries that omit it are treated as never-expiring, and an +// expired token is instead recovered by the client's 401 retry). +struct token_response { + std::string token; + std::optional expires_in; +}; + // --- network operations ------------------------------------------------- // Probe `/v2/` and parse the auth challenge. Returns the parsed @@ -39,8 +49,8 @@ discover_challenge(const std::string& registry_url); // Request a bearer token for the given scopes, optionally authenticating with // basic-auth credentials (required for push or private pull; omit for anonymous -// pull). Returns the raw token string for use as `Authorization: Bearer `. -util::expected +// pull). +util::expected fetch_token(const bearer_challenge& challenge, const std::vector& scopes, const std::optional& creds = std::nullopt); diff --git a/src/oci/client.cpp b/src/oci/client.cpp index 5f8a14f4..1361de93 100644 --- a/src/oci/client.cpp +++ b/src/oci/client.cpp @@ -93,7 +93,15 @@ client::token_for(bool write) { } auto& cache = write ? push_token_ : pull_token_; if (cache) { - return *cache; + if (!cache->expires_at || + std::chrono::steady_clock::now() < + cache->expires_at.value() - token_refresh_margin) { + return cache->value; + } + spdlog::debug("oci::token_for cached {} token reached its advertised " + "expiry, refreshing", + write ? "push" : "pull"); + cache = std::nullopt; } std::vector scopes; scopes.push_back( @@ -106,8 +114,57 @@ client::token_for(bool write) { if (!token) { return util::unexpected{token.error()}; } - cache = *token; - return *cache; + cached_token entry{.value = std::move(token->token), + .expires_at = std::nullopt}; + if (token->expires_in) { + entry.expires_at = std::chrono::steady_clock::now() + + std::chrono::seconds{token->expires_in.value()}; + spdlog::debug("oci::token_for fetched {} token (expires in {}s)", + write ? "push" : "pull", token->expires_in.value()); + } else { + spdlog::debug("oci::token_for fetched {} token (no advertised expiry)", + write ? "push" : "pull"); + } + cache = std::move(entry); + return cache->value; +} + +util::expected +client::authed_perform(util::curl::request& req, bool write, + const std::function& reset) { + auto& cache = write ? push_token_ : pull_token_; + const bool was_cached = cache.has_value(); + auto token = token_for(write); + if (!token) { + return util::unexpected{client_error{token.error()}}; + } + req.bearer_token = *token; + + auto resp = util::curl::perform(req); + + // a 401 on a token that was fetched for an earlier request usually means + // it expired in between (e.g. during a transfer that outlived the token's + // lifetime): fetch a fresh token and retry once. + if (resp && resp->status == 401 && was_cached && token->has_value()) { + spdlog::debug("oci::authed_perform 401 with a cached token: " + "refreshing the token and retrying {}", + req.url); + cache = std::nullopt; + token = token_for(write); + if (!token) { + return util::unexpected{client_error{token.error()}}; + } + req.bearer_token = *token; + if (reset) { + reset(); + } + resp = util::curl::perform(req); + } + + if (!resp) { + return util::unexpected{client_error{resp.error().message}}; + } + return *resp; } void client::add_pull_scope(std::string repository) { @@ -115,18 +172,13 @@ void client::add_pull_scope(std::string repository) { } util::expected client::blob_exists(const digest& d) { - auto token = token_for(false); - if (!token) { - return util::unexpected{client_error{token.error()}}; - } util::curl::request req; req.url = registry_url_ + detail::blob_path(repository_, d.string()); req.method = util::curl::http_method::head; - req.bearer_token = *token; - auto resp = util::curl::perform(req); + auto resp = authed_perform(req, false); if (!resp) { - return util::unexpected{client_error{resp.error().message}}; + return util::unexpected{resp.error()}; } if (resp->status == 200) { return true; @@ -141,19 +193,14 @@ util::expected client::blob_exists(const digest& d) { } util::expected client::get_blob(const digest& d) { - auto token = token_for(false); - if (!token) { - return util::unexpected{client_error{token.error()}}; - } util::curl::request req; req.url = registry_url_ + detail::blob_path(repository_, d.string()); - req.bearer_token = *token; // registries 307-redirect blob downloads to backing storage. req.follow_redirects = true; - auto resp = util::curl::perform(req); + auto resp = authed_perform(req, false); if (!resp) { - return util::unexpected{client_error{resp.error().message}}; + return util::unexpected{resp.error()}; } if (resp->status != 200) { return util::unexpected{client_error{ @@ -178,11 +225,6 @@ util::expected client::get_blob_to_file( file.string(), parent.string())}}; } - auto token = token_for(false); - if (!token) { - return util::unexpected{client_error{token.error()}}; - } - // stream to a temporary name and rename into place only after the // transfer and digest verification succeed: a failed or interrupted // download must never leave a file at the final path, where a later @@ -191,7 +233,6 @@ util::expected client::get_blob_to_file( util::curl::request req; req.url = registry_url_ + detail::blob_path(repository_, d.string()); - req.bearer_token = *token; req.follow_redirects = true; req.download_file = partial; req.on_download_progress = std::move(progress); @@ -227,9 +268,15 @@ util::expected client::get_blob_to_file( spdlog::debug("oci::get_blob_to_file downloading {} to {}", req.url, partial.string()); - auto resp = util::curl::perform(req); + // on a 401-retry the hash must restart from scratch: the retried request + // rewrites the partial file from the beginning ("wb" truncates it). + auto resp = authed_perform(req, false, [verify, &hash]() { + if (verify) { + util::sha256_init(hash); + } + }); if (!resp) { - return fail({resp.error().message}); + return fail(resp.error()); } if (resp->status != 200) { return fail( @@ -263,19 +310,14 @@ util::expected client::get_blob_to_file( util::expected client::get_manifest(const reference& ref) { - auto token = token_for(false); - if (!token) { - return util::unexpected{client_error{token.error()}}; - } util::curl::request req; req.url = registry_url_ + detail::manifest_path(repository_, ref.string()); - req.bearer_token = *token; req.header_lines = {fmt::format("Accept: {}", media_type_manifest), fmt::format("Accept: {}", media_type_index)}; - auto resp = util::curl::perform(req); + auto resp = authed_perform(req, false); if (!resp) { - return util::unexpected{client_error{resp.error().message}}; + return util::unexpected{resp.error()}; } if (resp->status != 200) { return util::unexpected{client_error{ @@ -296,17 +338,12 @@ client::get_manifest(const reference& ref) { } util::expected, client_error> client::list_tags() { - auto token = token_for(false); - if (!token) { - return util::unexpected{client_error{token.error()}}; - } util::curl::request req; req.url = registry_url_ + detail::tags_path(repository_); - req.bearer_token = *token; - auto resp = util::curl::perform(req); + auto resp = authed_perform(req, false); if (!resp) { - return util::unexpected{client_error{resp.error().message}}; + return util::unexpected{resp.error()}; } if (resp->status != 200) { return util::unexpected{client_error{ @@ -323,17 +360,12 @@ util::expected, client_error> client::list_tags() { util::expected, client_error> client::referrers(const digest& d) { - auto token = token_for(false); - if (!token) { - return util::unexpected{client_error{token.error()}}; - } util::curl::request req; req.url = registry_url_ + detail::referrers_path(repository_, d.string()); - req.bearer_token = *token; - auto resp = util::curl::perform(req); + auto resp = authed_perform(req, false); if (!resp) { - return util::unexpected{client_error{resp.error().message}}; + return util::unexpected{resp.error()}; } if (resp->status == 404) { // per the OCI 1.1 distribution spec, a 404 means the registry does @@ -362,23 +394,18 @@ client::referrers(const digest& d) { util::expected, client_error> client::referrers_from_tag(const digest& d) { - auto token = token_for(false); - if (!token) { - return util::unexpected{client_error{token.error()}}; - } // the tag schema names an OCI image index - in the same // repository as the subject (see maintain_referrers_tag in push.cpp). const auto tag = d.algorithm() + "-" + d.hex(); util::curl::request req; req.url = registry_url_ + detail::manifest_path(repository_, tag); - req.bearer_token = *token; req.header_lines = {fmt::format("Accept: {}", media_type_index), fmt::format("Accept: {}", media_type_manifest)}; spdlog::debug("oci::referrers_from_tag fetching {}", req.url); - auto resp = util::curl::perform(req); + auto resp = authed_perform(req, false); if (!resp) { - return util::unexpected{client_error{resp.error().message}}; + return util::unexpected{resp.error()}; } if (resp->status == 404) { // the tag was never created: the subject has no referrers. @@ -402,22 +429,18 @@ client::referrers_from_tag(const digest& d) { return *refs; } -namespace { - // drive the monolithic upload handshake: POST an upload session, then PUT the // payload (carried by `body_setup`) to the returned Location with ?digest=. -util::expected -do_put_blob(const std::string& registry_url, const std::string& uploads, - const std::optional& token, const std::string& digest, - const std::function& body_setup) { +util::expected client::put_blob_impl( + const std::string& digest, + const std::function& body_setup) { // 1. open an upload session util::curl::request post; - post.url = registry_url + uploads; + post.url = registry_url_ + detail::uploads_path(repository_); post.method = util::curl::http_method::post; - post.bearer_token = token; - auto opened = util::curl::perform(post); + auto opened = authed_perform(post, true); if (!opened) { - return util::unexpected{client_error{opened.error().message}}; + return util::unexpected{opened.error()}; } if (opened->status != 202) { return util::unexpected{client_error{ @@ -434,16 +457,15 @@ do_put_blob(const std::string& registry_url, const std::string& uploads, // 2. PUT the payload to ?digest= util::curl::request put; - put.url = detail::resolve_upload_url(registry_url, *location, digest); + put.url = detail::resolve_upload_url(registry_url_, *location, digest); put.method = util::curl::http_method::put; - put.bearer_token = token; put.header_lines = { fmt::format("Content-Type: {}", media_type_octet_stream)}; body_setup(put); - auto done = util::curl::perform(put); + auto done = authed_perform(put, true); if (!done) { - return util::unexpected{client_error{done.error().message}}; + return util::unexpected{done.error()}; } if (done->status != 201) { return util::unexpected{client_error{ @@ -455,26 +477,19 @@ do_put_blob(const std::string& registry_url, const std::string& uploads, return {}; } -} // namespace - util::expected client::mount_blob(const digest& d, const std::string& from_repository) { if (auto exists = blob_exists(d); exists && *exists) { return true; } - auto token = token_for(true); - if (!token) { - return util::unexpected{client_error{token.error()}}; - } util::curl::request post; post.url = registry_url_ + detail::uploads_path(repository_) + "?mount=" + d.string() + "&from=" + from_repository; post.method = util::curl::http_method::post; - post.bearer_token = *token; - auto resp = util::curl::perform(post); + auto resp = authed_perform(post, true); if (!resp) { - return util::unexpected{client_error{resp.error().message}}; + return util::unexpected{resp.error()}; } // 201: the blob was mounted. 202: the registry declined and opened an // upload session instead (caller must copy the blob the slow way). @@ -501,17 +516,13 @@ client::put_blob(const digest& d, const std::filesystem::path& file, spdlog::trace("oci::put_blob {} already present", d.string()); return {}; } - auto token = token_for(true); - if (!token) { - return util::unexpected{client_error{token.error()}}; - } - return do_put_blob(registry_url_, detail::uploads_path(repository_), *token, - d.string(), [&file, &progress](util::curl::request& r) { - r.upload_file = file; - if (progress) { - r.on_upload_progress = progress; - } - }); + return put_blob_impl(d.string(), + [&file, &progress](util::curl::request& r) { + r.upload_file = file; + if (progress) { + r.on_upload_progress = progress; + } + }); } util::expected @@ -519,32 +530,22 @@ client::put_blob_bytes(const digest& d, const std::string& data) { if (auto exists = blob_exists(d); exists && *exists) { return {}; } - auto token = token_for(true); - if (!token) { - return util::unexpected{client_error{token.error()}}; - } - return do_put_blob(registry_url_, detail::uploads_path(repository_), *token, - d.string(), - [&data](util::curl::request& r) { r.body = data; }); + return put_blob_impl(d.string(), + [&data](util::curl::request& r) { r.body = data; }); } util::expected client::put_manifest(const reference& ref, const std::string& body, std::string_view media_type) { - auto token = token_for(true); - if (!token) { - return util::unexpected{client_error{token.error()}}; - } util::curl::request req; req.url = registry_url_ + detail::manifest_path(repository_, ref.string()); req.method = util::curl::http_method::put; - req.bearer_token = *token; req.body = body; req.header_lines = {fmt::format("Content-Type: {}", media_type)}; - auto resp = util::curl::perform(req); + auto resp = authed_perform(req, true); if (!resp) { - return util::unexpected{client_error{resp.error().message}}; + return util::unexpected{resp.error()}; } if (resp->status != 201) { return util::unexpected{client_error{ diff --git a/src/oci/client.h b/src/oci/client.h index 4e19ef7b..5d856131 100644 --- a/src/oci/client.h +++ b/src/oci/client.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -12,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -166,25 +168,56 @@ class client { private: client() = default; - // return a bearer token for this repository, fetching/caching as needed. - // returns std::nullopt for an anonymous registry (no auth challenge), in - // which case requests are sent without an Authorization header. + // a cached bearer token and the deadline after which it is considered + // stale (absent when the token endpoint did not advertise a lifetime). + struct cached_token { + std::string value; + std::optional expires_at; + }; + + // refresh a cached token this long before its advertised expiry, so a + // token that is about to lapse is not used to start a new request. + static constexpr std::chrono::seconds token_refresh_margin{30}; + + // return a bearer token for this repository, fetching/caching as needed; + // a cached token past (or within token_refresh_margin of) its advertised + // expiry is refetched. returns std::nullopt for an anonymous registry (no + // auth challenge), in which case requests are sent without an + // Authorization header. util::expected, std::string> token_for(bool write); + // set the bearer token on `req` and perform it. if the registry rejects + // a previously cached token with 401 — typically one whose lifetime ran + // out during a preceding long transfer, without the token endpoint having + // advertised expires_in — a fresh token is fetched and the request + // retried once; `reset` (when set) runs before the retry so the caller + // can rewind per-attempt state (e.g. a streaming hash). a 401 on a + // freshly fetched token is a real denial and is returned as-is. + util::expected + authed_perform(util::curl::request& req, bool write, + const std::function& reset = {}); + // read the referrers of `d` from the referrers tag schema index // (-): the fallback for registries without the Referrers API. util::expected, client_error> referrers_from_tag(const digest& d); + // upload a blob via the monolithic POST-then-PUT handshake; `body_setup` + // attaches the payload (a file to stream, or an in-memory body) to the + // PUT request. + util::expected + put_blob_impl(const std::string& digest, + const std::function& body_setup); + std::string registry_url_; // normalised, no trailing '/' std::string repository_; std::optional creds_; // the auth challenge, or nullopt when the registry permits anonymous // access. std::optional challenge_; - std::optional pull_token_; - std::optional push_token_; + std::optional pull_token_; + std::optional push_token_; // extra repositories to request pull scope for (cross-repo mount). std::vector extra_pull_scopes_; }; diff --git a/src/oci/push.cpp b/src/oci/push.cpp index 12f411af..54904ec6 100644 --- a/src/oci/push.cpp +++ b/src/oci/push.cpp @@ -422,13 +422,15 @@ copy_image(const std::string& registry_base, const std::string& src_repo, } // copy referrers (the --recursive part): meta and any other attachments. + // referrers() already degrades gracefully on registries without the + // Referrers API (tag-schema fallback, empty list when nothing is + // attached), so any error here is a real failure — surface it rather + // than silently copying the image without its metadata. auto refs = src->referrers(manifest_digest); if (!refs) { - // a registry without the Referrers API just has nothing to recurse - // into. - spdlog::debug("copy_image: no referrers for {} ({})", - manifest_digest.string(), refs.error()); - return {}; + return util::unexpected{ + fmt::format("unable to list the artifacts attached to {}: {}", + manifest_digest.string(), refs.error().message)}; } for (const auto& r : *refs) { auto rm = src->get_manifest(reference::digest(r.digest)); @@ -456,6 +458,11 @@ copy_image(const std::string& registry_base, const std::string& src_repo, !ok) { return util::unexpected{ok.error().message}; } + // referrer manifests are pushed by digest only, which registries + // without the Referrers API do not index: maintain the tag-schema + // index on the destination so the attachment stays discoverable + // there too. + maintain_referrers_tag(*dst, manifest_digest, r); } return {}; diff --git a/src/oci/util.cpp b/src/oci/util.cpp index 92e8b203..6766df3f 100644 --- a/src/oci/util.cpp +++ b/src/oci/util.cpp @@ -126,18 +126,25 @@ std::string token_url(const bearer_challenge& challenge, return url; } -std::optional parse_token_response(std::string_view body) { +std::optional parse_token_response(std::string_view body) { auto j = nlohmann::json::parse(body, nullptr, /*allow_exceptions=*/false); if (j.is_discarded() || !j.is_object()) { return std::nullopt; } + token_response out; if (auto it = j.find("token"); it != j.end() && it->is_string()) { - return it->get(); + out.token = it->get(); + } else if (auto alt = j.find("access_token"); + alt != j.end() && alt->is_string()) { + out.token = alt->get(); + } else { + return std::nullopt; } - if (auto it = j.find("access_token"); it != j.end() && it->is_string()) { - return it->get(); + if (auto it = j.find("expires_in"); + it != j.end() && it->is_number_integer()) { + out.expires_in = it->get(); } - return std::nullopt; + return out; } std::string repository_scope(std::string_view repository, diff --git a/src/oci/util.h b/src/oci/util.h index 01a9ce30..40e9c294 100644 --- a/src/oci/util.h +++ b/src/oci/util.h @@ -56,9 +56,10 @@ std::optional> parse_referrers(std::string_view body); std::string token_url(const bearer_challenge& challenge, const std::vector& scopes); -// extract the token from a token endpoint response body ("token" or -// "access_token"). -std::optional parse_token_response(std::string_view body); +// extract the token ("token" or "access_token") and its advertised lifetime +// ("expires_in", when present as an integer) from a token endpoint response +// body. +std::optional parse_token_response(std::string_view body); // build a "repository::" scope string. std::string repository_scope(std::string_view repository, diff --git a/test/unit/oci_auth.cpp b/test/unit/oci_auth.cpp index bc6b3145..fb2bd9ea 100644 --- a/test/unit/oci_auth.cpp +++ b/test/unit/oci_auth.cpp @@ -34,14 +34,34 @@ TEST_CASE("oci token_url", "[oci][auth]") { } TEST_CASE("oci parse_token_response", "[oci][auth]") { - REQUIRE(parse_token_response(R"({"token":"abc123"})") == "abc123"); - REQUIRE(parse_token_response(R"({"access_token":"xyz"})") == "xyz"); + auto t = parse_token_response(R"({"token":"abc123"})"); + REQUIRE(t.has_value()); + REQUIRE(t->token == "abc123"); + REQUIRE(!t->expires_in.has_value()); + + auto a = parse_token_response(R"({"access_token":"xyz"})"); + REQUIRE(a.has_value()); + REQUIRE(a->token == "xyz"); + // token preferred when both present - REQUIRE(parse_token_response(R"({"token":"t","access_token":"a"})") == "t"); + auto both = parse_token_response(R"({"token":"t","access_token":"a"})"); + REQUIRE(both.has_value()); + REQUIRE(both->token == "t"); + + // the advertised lifetime is extracted when it is an integer + auto exp = parse_token_response(R"({"token":"t","expires_in":300})"); + REQUIRE(exp.has_value()); + REQUIRE(exp->expires_in == 300); + + // a non-integer expires_in is ignored, not an error + auto bad_exp = parse_token_response(R"({"token":"t","expires_in":"300"})"); + REQUIRE(bad_exp.has_value()); + REQUIRE(!bad_exp->expires_in.has_value()); + // missing / malformed - REQUIRE(parse_token_response(R"({"expires_in":300})") == std::nullopt); - REQUIRE(parse_token_response("not json") == std::nullopt); - REQUIRE(parse_token_response("") == std::nullopt); + REQUIRE(!parse_token_response(R"({"expires_in":300})").has_value()); + REQUIRE(!parse_token_response("not json").has_value()); + REQUIRE(!parse_token_response("").has_value()); } TEST_CASE("oci repository_scope", "[oci][auth]") { diff --git a/test/unit/oci_registry.cpp b/test/unit/oci_registry.cpp index c6be114a..a59ad16c 100644 --- a/test/unit/oci_registry.cpp +++ b/test/unit/oci_registry.cpp @@ -290,6 +290,14 @@ TEST_CASE("oci registry copy preserves digest", "[registry]") { auto pushed = oci::push_squashfs(*src, sqfs, oci::reference::tag("v1")); REQUIRE(pushed.has_value()); + // attach metadata to the source image: copy must carry it across. + const auto meta = dir / "meta"; + std::filesystem::create_directories(meta); + write_file(meta / "env.json", R"({"views":{}})"); + auto att = + oci::attach(*src, oci::reference::digest(*pushed), "uenv/meta", meta); + REQUIRE(att.has_value()); + auto copied = oci::copy_image(base, src_repo, dst_repo, *pushed, "v1", std::nullopt); REQUIRE(copied.has_value()); @@ -302,4 +310,24 @@ TEST_CASE("oci registry copy preserves digest", "[registry]") { const auto local = oci::digest::from_sha256(util::sha256_string(resp->body)); REQUIRE(local == *pushed); + + // the attached metadata is discoverable and pullable from the + // destination. + auto store = util::make_temp_dir().value(); + auto got = oci::pull_meta(*dst, *pushed, store); + REQUIRE(got.has_value()); + REQUIRE(*got); + REQUIRE(read_file(store / "meta" / "env.json") == R"({"views":{}})"); + + // copy must also recreate the referrers tag-schema index on the + // destination, so the attachment is discoverable on registries without + // the Referrers API. + const auto fallback_tag = + oci::reference::tag(pushed->algorithm() + "-" + pushed->hex()); + auto tag_index = dst->get_manifest(fallback_tag); + REQUIRE(tag_index.has_value()); + auto tag_refs = oci::detail::parse_referrers(tag_index->body); + REQUIRE(tag_refs.has_value()); + REQUIRE(tag_refs->size() == 1); + REQUIRE(tag_refs->front().artifact_type == "uenv/meta"); } From f100bbe70e49bd16a1f54d591ae750a2f3c2fbb4 Mon Sep 17 00:00:00 2001 From: bcumming Date: Fri, 10 Jul 2026 15:55:20 +0200 Subject: [PATCH 25/29] bugfix: wrong-typed JSON in registry responses no longer aborts via uncaught type_error --- src/oci/manifest.cpp | 22 ++++++++++++++-------- src/oci/util.cpp | 22 +++++++++++++++++++--- src/oci/util.h | 17 +++++++++++++++++ test/unit/oci_client.cpp | 12 ++++++++++++ test/unit/oci_manifest.cpp | 19 +++++++++++++++++++ 5 files changed, 81 insertions(+), 11 deletions(-) diff --git a/src/oci/manifest.cpp b/src/oci/manifest.cpp index d7a6397a..a8276507 100644 --- a/src/oci/manifest.cpp +++ b/src/oci/manifest.cpp @@ -7,6 +7,7 @@ #include #include #include +#include namespace oci { @@ -130,13 +131,13 @@ namespace { util::expected parse_descriptor(const nlohmann::json& j) { - auto dg = digest::parse(j.value("digest", std::string{})); + auto dg = digest::parse(detail::json_string_or(j, "digest", {})); if (!dg) { return util::unexpected{dg.error().message()}; } - descriptor d{.media_type = j.value("mediaType", std::string{}), + descriptor d{.media_type = detail::json_string_or(j, "mediaType", {}), .digest = *dg, - .size = j.value("size", std::size_t{0})}; + .size = detail::json_size_or(j, "size", 0)}; if (auto a = j.find("artifactType"); a != j.end() && a->is_string()) { d.artifact_type = a->get(); } @@ -167,8 +168,9 @@ util::expected parse_manifest(std::string_view body) { } manifest m; - m.media_type = j.value("mediaType", std::string{media_type_manifest}); - m.artifact_type = j.value("artifactType", std::string{}); + m.media_type = detail::json_string_or(j, "mediaType", + std::string{media_type_manifest}); + m.artifact_type = detail::json_string_or(j, "artifactType", {}); // reject multi-arch image indexes / manifest lists: they carry a // `manifests` array of per-platform descriptors instead of `layers`, so @@ -193,15 +195,19 @@ util::expected parse_manifest(std::string_view body) { if (auto ls = j.find("layers"); ls != j.end() && ls->is_array()) { for (const auto& l : *ls) { - auto dg = digest::parse(l.value("digest", std::string{})); + if (!l.is_object()) { + return util::unexpected{ + "manifest layer entry is not a JSON object"}; + } + auto dg = digest::parse(detail::json_string_or(l, "digest", {})); if (!dg) { return util::unexpected{fmt::format("invalid layer digest: {}", dg.error().message())}; } manifest_layer layer{.media_type = - l.value("mediaType", std::string{}), + detail::json_string_or(l, "mediaType", {}), .digest = *dg, - .size = l.value("size", std::size_t{0}), + .size = detail::json_size_or(l, "size", 0), .annotations = parse_annotations(l)}; m.layers.push_back(std::move(layer)); } diff --git a/src/oci/util.cpp b/src/oci/util.cpp index 6766df3f..67b4b197 100644 --- a/src/oci/util.cpp +++ b/src/oci/util.cpp @@ -61,6 +61,22 @@ std::string resolve_upload_url(std::string_view registry_url, return url; } +std::string json_string_or(const nlohmann::json& j, const char* key, + std::string fallback) { + if (auto it = j.find(key); it != j.end() && it->is_string()) { + return it->get(); + } + return fallback; +} + +std::size_t json_size_or(const nlohmann::json& j, const char* key, + std::size_t fallback) { + if (auto it = j.find(key); it != j.end() && it->is_number_unsigned()) { + return it->get(); + } + return fallback; +} + std::optional> parse_tags_list(std::string_view body) { auto j = nlohmann::json::parse(body, nullptr, /*allow_exceptions=*/false); if (j.is_discarded() || !j.is_object()) { @@ -92,13 +108,13 @@ std::optional> parse_referrers(std::string_view body) { continue; } // a descriptor must carry a valid digest; skip malformed entries. - auto dg = digest::parse(m.value("digest", std::string{})); + auto dg = digest::parse(json_string_or(m, "digest", {})); if (!dg) { continue; } - descriptor d{.media_type = m.value("mediaType", std::string{}), + descriptor d{.media_type = json_string_or(m, "mediaType", {}), .digest = *dg, - .size = m.value("size", std::size_t{0})}; + .size = json_size_or(m, "size", 0)}; if (auto a = m.find("artifactType"); a != m.end() && a->is_string()) { d.artifact_type = a->get(); } diff --git a/src/oci/util.h b/src/oci/util.h index 40e9c294..67189916 100644 --- a/src/oci/util.h +++ b/src/oci/util.h @@ -6,11 +6,14 @@ // client.h/auth.h interfaces — include this header only from src/oci/*.cpp // and test/unit/oci_*.cpp. +#include #include #include #include #include +#include + #include #include @@ -38,6 +41,20 @@ std::string resolve_upload_url(std::string_view registry_url, std::string_view location, std::string_view digest); +// --- checked JSON field accessors -------------------------------------------- +// Registry responses are untrusted input, and nlohmann's value() throws +// json::type_error when a key is present with a mismatched type. These return +// the fallback instead. + +// value of the string field `key`, or `fallback` when absent or not a string. +std::string json_string_or(const nlohmann::json& j, const char* key, + std::string fallback); + +// value of the non-negative integer field `key`, or `fallback` when absent or +// not such a number. +std::size_t json_size_or(const nlohmann::json& j, const char* key, + std::size_t fallback); + // --- response-body parsers ------------------------------------------------- // parse a /v2//tags/list response body. diff --git a/test/unit/oci_client.cpp b/test/unit/oci_client.cpp index 335b6ee9..73224621 100644 --- a/test/unit/oci_client.cpp +++ b/test/unit/oci_client.cpp @@ -91,6 +91,18 @@ TEST_CASE("oci parse_referrers", "[oci][client]") { REQUIRE(skipped.has_value()); REQUIRE(skipped->empty()); + // wrong-typed fields never throw: a non-string digest skips the entry, + // and a non-integer size falls back to 0 + auto wrong_digest = parse_referrers(R"({"manifests":[{"digest":123}]})"); + REQUIRE(wrong_digest.has_value()); + REQUIRE(wrong_digest->empty()); + auto wrong_size = parse_referrers( + fmt::format(R"({{"manifests":[{{"digest":"sha256:{}","size":"5"}}]}})", + digest_hex)); + REQUIRE(wrong_size.has_value()); + REQUIRE(wrong_size->size() == 1); + REQUIRE(wrong_size->front().size == 0); + // no manifests array -> empty list auto empty = parse_referrers(R"({"manifests":[]})"); REQUIRE(empty.has_value()); diff --git a/test/unit/oci_manifest.cpp b/test/unit/oci_manifest.cpp index 6a380ce7..7d1fd49b 100644 --- a/test/unit/oci_manifest.cpp +++ b/test/unit/oci_manifest.cpp @@ -125,6 +125,25 @@ TEST_CASE("parse_manifest rejects malformed", "[oci][manifest]") { .has_value()); } +TEST_CASE("parse_manifest survives wrong-typed fields", "[oci][manifest]") { + // registry responses are untrusted: a key that is present with the wrong + // type must produce a parse error (or a fallback), never an uncaught + // nlohmann::json::type_error. + REQUIRE_FALSE(parse_manifest(R"({"layers":[{"digest":123}]})").has_value()); + REQUIRE_FALSE(parse_manifest(R"({"layers":[1]})").has_value()); + // a wrong-typed config size falls back to 0 (the digest is valid) + const auto cfg = fmt::format( + R"({{"config":{{"digest":"sha256:{}","size":"2"}}}})", + "f7f04f3b2cf562336c73542f0c53503c3b853ac459f081878843f878955cf267"); + auto parsed = parse_manifest(cfg); + REQUIRE(parsed.has_value()); + REQUIRE(parsed->config.size == 0); + // a wrong-typed top-level mediaType falls back to the manifest media type + auto media = parse_manifest(R"({"mediaType":123})"); + REQUIRE(media.has_value()); + REQUIRE(media->media_type == media_type_manifest); +} + TEST_CASE("parse_manifest rejects image indexes", "[oci][manifest]") { // by media type REQUIRE_FALSE( From 007996001aaad449a203e81b7f12dec551093955 Mon Sep 17 00:00:00 2001 From: bcumming Date: Fri, 10 Jul 2026 16:28:19 +0200 Subject: [PATCH 26/29] bugfix: verify the digest of downloaded meta archives, and stop buffering them in memory pull_meta fetched the meta tar.gz with the in-memory get_blob, which checks only the HTTP status and performs no sha256 verification, then wrote the buffer back out to a temp file for tar. The meta payload (env.json, views) is later sourced into user environments, and it was the only downloaded artifact that skipped verification. Stream the layer straight to the staging path with get_blob_to_file, which hashes during download and rejects a digest mismatch. This also removes a redundant RAM+disk round trip of the whole blob. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/oci/pull.cpp | 55 ++++++++++++++++++------------------------------ 1 file changed, 20 insertions(+), 35 deletions(-) diff --git a/src/oci/pull.cpp b/src/oci/pull.cpp index abe78306..77287ec2 100644 --- a/src/oci/pull.cpp +++ b/src/oci/pull.cpp @@ -1,5 +1,4 @@ #include -#include #include #include @@ -22,39 +21,16 @@ namespace { // title annotation oras gives the squashfs layer. constexpr std::string_view squashfs_title = "store.squashfs"; -// extract a gzipped tar (held in memory) into `dest`, using the system tar. -util::expected extract_targz(const std::string& data, +// extract a gzipped tar into `dest`, using the system tar. +util::expected extract_targz(const fs::path& archive, const fs::path& dest) { - // stage the archive in a private temp dir (unique name, not a predictable - // path inside `dest`) before handing it to tar. - auto work = util::make_temp_dir(); - if (!work) { - return util::unexpected{work.error()}; - } - auto tmp = *work / "meta.tar.gz"; - { - std::ofstream out(tmp, std::ios::binary | std::ios::trunc); - if (!out) { - return util::unexpected{ - fmt::format("unable to write temporary {}", tmp.string())}; - } - out.write(data.data(), static_cast(data.size())); - if (!out) { - return util::unexpected{ - fmt::format("error writing temporary {}", tmp.string())}; - } - } - - std::error_code ec; - auto proc = util::run({"tar", "-xzf", tmp.string(), "-C", dest.string()}); + auto proc = + util::run({"tar", "-xzf", archive.string(), "-C", dest.string()}); if (!proc) { - fs::remove(tmp, ec); return util::unexpected{ fmt::format("unable to run tar to unpack meta: {}", proc.error())}; } - auto rc = proc->wait(); - fs::remove(tmp, ec); - if (rc != 0) { + if (auto rc = proc->wait(); rc != 0) { return util::unexpected{ fmt::format("tar failed to unpack meta (exit {})", rc)}; } @@ -133,15 +109,24 @@ pull_meta(client& c, const digest& manifest_digest, const fs::path& store) { } spdlog::debug("oci::pull_meta layer {}", layer->digest.string()); - auto blob = c.get_blob(layer->digest); - if (!blob) { - return util::unexpected{blob.error().message}; - } - if (auto ok = util::ensure_directory(store); !ok) { return util::unexpected{ok.error()}; } - if (auto ok = extract_targz(*blob, store); !ok) { + + // stage the archive in a private temp dir (unique name, not a predictable + // path inside `store`) before handing it to tar; make_temp_dir registers it + // for removal on exit. get_blob_to_file verifies the digest as it streams: + // the extracted meta (env.json, views) is later sourced into user + // environments, so it must not be taken on trust. + auto work = util::make_temp_dir(); + if (!work) { + return util::unexpected{work.error()}; + } + const auto archive = work.value() / "meta.tar.gz"; + if (auto ok = c.get_blob_to_file(layer->digest, archive); !ok) { + return util::unexpected{ok.error().message}; + } + if (auto ok = extract_targz(archive, store); !ok) { return util::unexpected{ok.error()}; } return true; From 09a6ea78b7e9b1d952154776537a17d1b788a031 Mon Sep 17 00:00:00 2001 From: bcumming Date: Fri, 10 Jul 2026 16:31:20 +0200 Subject: [PATCH 27/29] remove stale get_blob interface --- src/oci/client.cpp | 19 ------------------- src/oci/client.h | 3 --- 2 files changed, 22 deletions(-) diff --git a/src/oci/client.cpp b/src/oci/client.cpp index 1361de93..8e445f2b 100644 --- a/src/oci/client.cpp +++ b/src/oci/client.cpp @@ -192,25 +192,6 @@ util::expected client::blob_exists(const digest& d) { resp->status}}; } -util::expected client::get_blob(const digest& d) { - util::curl::request req; - req.url = registry_url_ + detail::blob_path(repository_, d.string()); - // registries 307-redirect blob downloads to backing storage. - req.follow_redirects = true; - - auto resp = authed_perform(req, false); - if (!resp) { - return util::unexpected{resp.error()}; - } - if (resp->status != 200) { - return util::unexpected{client_error{ - fmt::format("failed to fetch blob {} (status {}): {}", d.string(), - resp->status, util::curl::http_message(resp->status)), - resp->status}}; - } - return resp->body; -} - util::expected client::get_blob_to_file( const digest& d, const std::filesystem::path& file, std::function progress, diff --git a/src/oci/client.h b/src/oci/client.h index 5d856131..da80ebe7 100644 --- a/src/oci/client.h +++ b/src/oci/client.h @@ -102,9 +102,6 @@ class client { // does a blob exist? HEAD; 200 -> true, 404 -> false. util::expected blob_exists(const digest& d); - // fetch a blob's bytes (follows the 307 redirect to backing storage). - util::expected get_blob(const digest& d); - // stream a blob straight to a file, never holding it in memory (for the // multi-GB squashfs layer). follows the 307 redirect to backing storage. // an optional progress callback receives (bytes_downloaded, bytes_total); From d78a59bd5679e85c09f4879d1085092d8c6d8df5 Mon Sep 17 00:00:00 2001 From: bcumming Date: Fri, 10 Jul 2026 16:46:22 +0200 Subject: [PATCH 28/29] use a docker-credential helper instaed of silently falling back to anonymous access --- src/oci/auth.cpp | 95 ++++++++++++++--- src/oci/util.cpp | 62 +++++++++++ src/oci/util.h | 19 ++++ src/util/subprocess.cpp | 7 ++ src/util/subprocess.h | 5 + test/unit/oci_auth.cpp | 218 ++++++++++++++++++++++++++++++++++++++- test/unit/subprocess.cpp | 48 +++++++++ 7 files changed, 435 insertions(+), 19 deletions(-) diff --git a/src/oci/auth.cpp b/src/oci/auth.cpp index 0e027023..ecc065ec 100644 --- a/src/oci/auth.cpp +++ b/src/oci/auth.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include @@ -19,6 +20,7 @@ #include #include #include +#include namespace oci { @@ -177,17 +179,75 @@ creds_from_token(std::string token, "provide a username with --username for the token."}; } -// normalise a docker config.json `auths` key (or a registry host) down to a -// bare host[:port] for comparison: drop any scheme and any trailing path. -std::string normalise_host(std::string_view key) { - auto s = key; - if (auto p = s.find("://"); p != std::string_view::npos) { - s.remove_prefix(p + 3); +// A credential helper that exits without consuming its stdin leaves us writing +// the server URL into a pipe with no reader. Short writes still land in the +// pipe buffer, but one large enough to overrun it raises SIGPIPE, which is +// fatal by default. +class sigpipe_guard { + void (*previous_)(int); + + public: + sigpipe_guard() : previous_(std::signal(SIGPIPE, SIG_IGN)) { + } + ~sigpipe_guard() { + std::signal(SIGPIPE, previous_); } - if (auto p = s.find('/'); p != std::string_view::npos) { - s = s.substr(0, p); + sigpipe_guard(const sigpipe_guard&) = delete; + sigpipe_guard& operator=(const sigpipe_guard&) = delete; +}; + +// the registry key to hand the helper on stdin. Helpers are keyed by the string +// the user logged in with, which may carry a scheme and path (docker hub stores +// "https://index.docker.io/v1/"), so prefer a configured key over the bare +// host. +std::string server_key(const nlohmann::json& cfg, const std::string& want) { + for (const auto* section : {"credHelpers", "auths"}) { + if (auto s = cfg.find(section); s != cfg.end() && s->is_object()) { + for (const auto& entry : s->items()) { + if (detail::normalise_host(entry.key()) == want) { + return entry.key(); + } + } + } } - return std::string{s}; + return want; +} + +// query a docker credential helper: `docker-credential- get` takes the +// registry on stdin and answers with a JSON credential on stdout. +util::expected, std::string> +creds_from_helper(const std::string& helper, const std::string& server) { + const auto program = fmt::format("docker-credential-{}", helper); + // WARNING: the helper's stdout carries a secret. Never log it. + auto proc = util::run({program, "get"}); + if (!proc) { + return util::unexpected{ + fmt::format("unable to run the credential helper {}: {}", program, + proc.error())}; + } + { + sigpipe_guard guard; + proc->in.putline(server); + // the helper reads stdin to EOF before it replies. + proc->in.close(); + } + const auto out = proc->out.string(); + const auto err = proc->err.string(); + + if (proc->wait() != 0) { + // the helper's way of saying it holds nothing for this registry; fall + // through to anonymous access rather than failing the command. + if (err.find("credentials not found") != std::string::npos) { + spdlog::debug("oci::creds_from_helper {} has no credential for {}", + program, server); + return std::nullopt; + } + return util::unexpected{ + fmt::format("the credential helper {} failed (exit {}): {}", + program, proc->rvalue(), util::strip(err))}; + } + spdlog::debug("oci::creds_from_helper {} answered for {}", program, server); + return detail::parse_helper_output(out); } // look up credentials for `host` in a docker config.json file. Returns nullopt @@ -206,21 +266,26 @@ creds_from_docker_config(const std::filesystem::path& cfg, return util::unexpected{ fmt::format("could not parse docker config {}", cfg.string())}; } + const std::string want = detail::normalise_host(host); + + // docker consults a credential helper before the inline auths entry: a + // `docker login` backed by a credential store leaves auths[host] empty. + if (auto helper = detail::helper_for_host(j, want)) { + return creds_from_helper(*helper, server_key(j, want)); + } + auto auths = j.find("auths"); if (auths == j.end() || !auths->is_object()) { return std::nullopt; } - const std::string want = normalise_host(host); for (const auto& [key, entry] : auths->items()) { - if (normalise_host(key) != want || !entry.is_object()) { + if (detail::normalise_host(key) != want || !entry.is_object()) { continue; } auto a = entry.find("auth"); if (a == entry.end() || !a->is_string()) { - // credsStore / credHelpers / identitytoken entries need an external - // helper we do not implement. - spdlog::warn("docker config entry for {} has no 'auth' field " - "(credential helpers are not supported)", + spdlog::warn("docker config entry for {} has no 'auth' field and " + "no credential helper is configured for it", want); return std::nullopt; } diff --git a/src/oci/util.cpp b/src/oci/util.cpp index 67b4b197..84dcd40f 100644 --- a/src/oci/util.cpp +++ b/src/oci/util.cpp @@ -163,6 +163,68 @@ std::optional parse_token_response(std::string_view body) { return out; } +std::string normalise_host(std::string_view key) { + auto s = key; + if (auto p = s.find("://"); p != std::string_view::npos) { + s.remove_prefix(p + 3); + } + if (auto p = s.find('/'); p != std::string_view::npos) { + s = s.substr(0, p); + } + return std::string{s}; +} + +std::optional helper_for_host(const nlohmann::json& cfg, + std::string_view host) { + const auto want = normalise_host(host); + if (auto helpers = cfg.find("credHelpers"); + helpers != cfg.end() && helpers->is_object()) { + for (const auto& entry : helpers->items()) { + if (normalise_host(entry.key()) == want && + entry.value().is_string()) { + return entry.value().get(); + } + } + } + if (auto store = cfg.find("credsStore"); + store != cfg.end() && store->is_string()) { + if (auto name = store->get(); !name.empty()) { + return name; + } + } + return std::nullopt; +} + +util::expected, std::string> +parse_helper_output(std::string_view body) { + auto j = nlohmann::json::parse(body, nullptr, /*allow_exceptions=*/false); + if (j.is_discarded() || !j.is_object()) { + return util::unexpected{ + "the credential helper returned malformed JSON"}; + } + auto username = json_string_or(j, "Username", ""); + auto secret = json_string_or(j, "Secret", ""); + + // a helper with nothing stored for the registry answers with empty fields. + if (username.empty() && secret.empty()) { + return std::nullopt; + } + // "" is the docker sentinel for an identity (refresh) token in + // Secret. Redeeming one needs the OAuth2 refresh_token grant, which the + // token handshake in this module does not implement. + if (username == "") { + return util::unexpected{ + "the credential helper returned an identity token, which uenv " + "cannot redeem; pass the token directly with --token"}; + } + if (username.empty() || secret.empty()) { + return util::unexpected{ + "the credential helper returned an incomplete credential"}; + } + return credentials{.username = std::move(username), + .password = std::move(secret)}; +} + std::string repository_scope(std::string_view repository, std::string_view actions) { return fmt::format("repository:{}:{}", repository, actions); diff --git a/src/oci/util.h b/src/oci/util.h index 67189916..44fd033d 100644 --- a/src/oci/util.h +++ b/src/oci/util.h @@ -82,5 +82,24 @@ std::optional parse_token_response(std::string_view body); std::string repository_scope(std::string_view repository, std::string_view actions); +// --- docker config.json helpers --------------------------------------------- + +// normalise a docker config.json `auths`/`credHelpers` key (or a registry host) +// down to a bare host[:port] for comparison: drop any scheme and any trailing +// path. +std::string normalise_host(std::string_view key); + +// the credential helper configured for `host` in a parsed docker config.json. +// A per-registry `credHelpers` entry wins over the global `credsStore`, as in +// docker itself. Returns nullopt when neither names a helper. +std::optional helper_for_host(const nlohmann::json& cfg, + std::string_view host); + +// parse the stdout of `docker-credential- get`, which is +// {"ServerURL":..,"Username":..,"Secret":..}. Returns nullopt when the helper +// holds no credential for the registry (it reports empty fields). +util::expected, std::string> +parse_helper_output(std::string_view body); + } // namespace detail } // namespace oci diff --git a/src/util/subprocess.cpp b/src/util/subprocess.cpp index f82f3080..9fb79fca 100644 --- a/src/util/subprocess.cpp +++ b/src/util/subprocess.cpp @@ -154,4 +154,11 @@ void buffered_ostream::putline(std::string_view line) { stream() << line << std::endl; } +void buffered_ostream::close() { + if (buffer_->is_open()) { + stream_->flush(); + buffer_->close(); + } +} + } // namespace util diff --git a/src/util/subprocess.h b/src/util/subprocess.h index ebf30fdf..6ac1cf9a 100644 --- a/src/util/subprocess.h +++ b/src/util/subprocess.h @@ -65,6 +65,11 @@ class buffered_ostream { std::ostream& stream(); void putline(std::string_view line); + + // flush and close the pipe, so the child reading this stream sees EOF. + // Required by any protocol where the child consumes all of stdin before + // replying; idempotent, and also performed on destruction. + void close(); }; enum class proc_status { running, finished }; diff --git a/test/unit/oci_auth.cpp b/test/unit/oci_auth.cpp index fb2bd9ea..a6dc628d 100644 --- a/test/unit/oci_auth.cpp +++ b/test/unit/oci_auth.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -76,6 +77,30 @@ void write_file(const std::filesystem::path& path, std::string_view content) { std::ofstream f{path}; f << content; } + +// Install an executable `docker-credential-` in `dir` and prepend `dir` +// to PATH for the lifetime of the guard, so that the execvp performed by +// creds_from_helper finds it. +class helper_on_path { + std::string previous_; + + public: + helper_on_path(const std::filesystem::path& dir, std::string_view name, + std::string_view script) { + const auto exe = dir / fmt::format("docker-credential-{}", name); + write_file(exe, script); + std::filesystem::permissions(exe, std::filesystem::perms::owner_all); + const auto* path = std::getenv("PATH"); + previous_ = path ? path : ""; + ::setenv("PATH", fmt::format("{}:{}", dir.string(), previous_).c_str(), + 1); + } + ~helper_on_path() { + ::setenv("PATH", previous_.c_str(), 1); + } + helper_on_path(const helper_on_path&) = delete; + helper_on_path& operator=(const helper_on_path&) = delete; +}; } // namespace TEST_CASE("oci get_credentials reads a token file", "[oci][auth]") { @@ -248,12 +273,11 @@ TEST_CASE("oci resolve_credentials precedence", "[oci][auth]") { REQUIRE_FALSE(c->has_value()); } - // a credsStore-only entry (no `auth`) is skipped, not an error. - SECTION("docker config credsStore entry skipped") { + // an entry with no `auth` and no helper configured for it -> anonymous. + SECTION("docker config entry with no auth and no helper") { auto dir = util::make_temp_dir().value(); auto cfg = dir / "config.json"; - write_file( - cfg, R"({"auths":{"reg.example.com":{}},"credsStore":"desktop"})"); + write_file(cfg, R"({"auths":{"reg.example.com":{}}})"); oci::credential_sources src; src.docker_config = cfg; auto c = oci::resolve_credentials(host, src); @@ -272,3 +296,189 @@ TEST_CASE("oci resolve_credentials precedence", "[oci][auth]") { REQUIRE_FALSE(c); } } + +TEST_CASE("oci helper_for_host", "[oci][auth]") { + using oci::detail::helper_for_host; + + // a per-registry credHelpers entry wins over the global credsStore. + auto both = nlohmann::json::parse( + R"({"credsStore":"global","credHelpers":{"reg.example.com":"perreg"}})"); + REQUIRE(helper_for_host(both, "reg.example.com") == "perreg"); + // ... and the credsStore still covers every other registry. + REQUIRE(helper_for_host(both, "other.example.com") == "global"); + + // credHelpers keys may carry a scheme and path, as auths keys do. + auto scheme = nlohmann::json::parse( + R"({"credHelpers":{"https://reg.example.com/v2/":"perreg"}})"); + REQUIRE(helper_for_host(scheme, "reg.example.com") == "perreg"); + + // no helper configured. + auto none = nlohmann::json::parse(R"({"auths":{"reg.example.com":{}}})"); + REQUIRE_FALSE(helper_for_host(none, "reg.example.com")); + + // an empty credsStore does not name a helper. + auto empty = nlohmann::json::parse(R"({"credsStore":""})"); + REQUIRE_FALSE(helper_for_host(empty, "reg.example.com")); + + // wrong-typed values must not throw. + auto typed = nlohmann::json::parse(R"({"credsStore":42,"credHelpers":[]})"); + REQUIRE_FALSE(helper_for_host(typed, "reg.example.com")); +} + +TEST_CASE("oci parse_helper_output", "[oci][auth]") { + using oci::detail::parse_helper_output; + + SECTION("a stored credential") { + auto c = parse_helper_output( + R"({"ServerURL":"reg.example.com","Username":"dave","Secret":"s3cret"})"); + REQUIRE(c); + REQUIRE(c->has_value()); + REQUIRE((*c)->username == "dave"); + REQUIRE((*c)->password == "s3cret"); + } + + // a helper holding nothing for the registry answers with empty fields. + SECTION("no stored credential is anonymous") { + auto c = parse_helper_output( + R"({"ServerURL":"reg.example.com","Username":"","Secret":""})"); + REQUIRE(c); + REQUIRE_FALSE(c->has_value()); + } + + // "" means Secret is an identity token, which needs the OAuth2 + // refresh_token grant that fetch_token does not implement. + SECTION("an identity token is a clear error") { + auto c = + parse_helper_output(R"({"Username":"","Secret":"refresh"})"); + REQUIRE_FALSE(c); + REQUIRE_THAT(c.error(), Catch::Matchers::ContainsSubstring("--token")); + } + + SECTION("a half-filled credential is an error") { + REQUIRE_FALSE( + parse_helper_output(R"({"Username":"dave","Secret":""})")); + REQUIRE_FALSE(parse_helper_output(R"({"Username":"","Secret":"pw"})")); + } + + SECTION("malformed output is an error, not a crash") { + REQUIRE_FALSE(parse_helper_output("not json at all")); + REQUIRE_FALSE(parse_helper_output("[]")); + } + + // wrong-typed fields must not throw json::type_error. + SECTION("wrong-typed fields are treated as absent") { + auto c = parse_helper_output(R"({"Username":42,"Secret":true})"); + REQUIRE(c); + REQUIRE_FALSE(c->has_value()); + } +} + +TEST_CASE("oci resolve_credentials via a docker credential helper", + "[oci][auth]") { + const std::string host = "reg.example.com"; + auto dir = util::make_temp_dir().value(); + const auto cfg = dir / "config.json"; + oci::credential_sources src; + src.docker_config = cfg; + + // `$(cat)` consumes stdin to EOF, so these also prove that the write end of + // the child's stdin is closed after the server URL is sent. + SECTION("credsStore supplies the credential") { + helper_on_path helper( + dir, "uenvtest", + "#!/bin/sh\nserver=$(cat)\n" + R"(printf '{"ServerURL":"%s","Username":"dave","Secret":"s3cret"}' "$server")" + "\n"); + write_file( + cfg, R"({"auths":{"reg.example.com":{}},"credsStore":"uenvtest"})"); + auto c = oci::resolve_credentials(host, src); + REQUIRE(c); + REQUIRE(c->has_value()); + REQUIRE((*c)->username == "dave"); + REQUIRE((*c)->password == "s3cret"); + } + + // the helper is keyed by the string the user logged in with, so the + // configured key is passed on stdin rather than the bare host. + SECTION("the configured registry key is passed to the helper") { + helper_on_path helper( + dir, "uenvecho", + "#!/bin/sh\nserver=$(cat)\n" + R"(printf '{"Username":"%s","Secret":"x"}' "$server")" + "\n"); + write_file( + cfg, + R"({"auths":{"https://reg.example.com/v2/":{}},"credsStore":"uenvecho"})"); + auto c = oci::resolve_credentials(host, src); + REQUIRE(c); + REQUIRE(c->has_value()); + REQUIRE((*c)->username == "https://reg.example.com/v2/"); + } + + SECTION("credHelpers overrides credsStore") { + helper_on_path global(dir, "uenvglobal", + "#!/bin/sh\ncat >/dev/null\n" + R"(printf '{"Username":"global","Secret":"x"}')" + "\n"); + helper_on_path perreg(dir, "uenvperreg", + "#!/bin/sh\ncat >/dev/null\n" + R"(printf '{"Username":"perreg","Secret":"x"}')" + "\n"); + write_file( + cfg, + R"({"credsStore":"uenvglobal","credHelpers":{"reg.example.com":"uenvperreg"}})"); + auto c = oci::resolve_credentials(host, src); + REQUIRE(c); + REQUIRE(c->has_value()); + REQUIRE((*c)->username == "perreg"); + } + + // "credentials not found" is how a helper reports an unknown registry. + SECTION("a helper with no stored credential is anonymous") { + helper_on_path helper( + dir, "uenvempty", + "#!/bin/sh\ncat >/dev/null\n" + "echo 'credentials not found in native keychain' >&2\nexit 1\n"); + write_file(cfg, R"({"credsStore":"uenvempty"})"); + auto c = oci::resolve_credentials(host, src); + REQUIRE(c); + REQUIRE_FALSE(c->has_value()); + } + + // any other failure is surfaced, rather than silently degrading. + SECTION("a failing helper is an error") { + helper_on_path helper(dir, "uenvbroken", + "#!/bin/sh\ncat >/dev/null\n" + "echo 'the keychain is locked' >&2\nexit 2\n"); + write_file(cfg, R"({"credsStore":"uenvbroken"})"); + auto c = oci::resolve_credentials(host, src); + REQUIRE_FALSE(c); + REQUIRE_THAT(c.error(), Catch::Matchers::ContainsSubstring( + "the keychain is locked")); + } + + // execvp fails in the forked child, which exits non-zero having read + // nothing. + SECTION("a helper missing from PATH is an error") { + write_file(cfg, R"({"credsStore":"uenv-definitely-not-installed"})"); + auto c = oci::resolve_credentials(host, src); + REQUIRE_FALSE(c); + REQUIRE_THAT(c.error(), + Catch::Matchers::ContainsSubstring( + "docker-credential-uenv-definitely-not-installed")); + } + + // A helper that exits without reading leaves the parent writing into a pipe + // with no reader. Writes below the 64KiB pipe buffer complete regardless, + // so pin the SIGPIPE guard with a registry key large enough to overrun it: + // without the guard the write kills the process with signal 13. + SECTION("a helper that never reads stdin does not kill the process") { + helper_on_path helper(dir, "uenvdeaf", "#!/bin/sh\nexit 0\n"); + const std::string padding(200000, 'x'); + write_file(cfg, fmt::format(R"({{"auths":{{"https://{}/{}":{{}}}},)" + R"("credsStore":"uenvdeaf"}})", + host, padding)); + auto c = oci::resolve_credentials(host, src); + REQUIRE_FALSE(c); // the helper answers nothing, which is an error + } +} diff --git a/test/unit/subprocess.cpp b/test/unit/subprocess.cpp index 6987c445..dceb2998 100644 --- a/test/unit/subprocess.cpp +++ b/test/unit/subprocess.cpp @@ -65,6 +65,54 @@ TEST_CASE("kill", "[subprocess]") { REQUIRE(proc->rvalue() == 9); } +TEST_CASE("stdin", "[subprocess]") { + // `cat` runs until its stdin reaches EOF, so each of these would hang + // forever if close() failed to close the underlying pipe. + { + auto proc = util::run({"cat"}); + REQUIRE(proc); + proc->in.putline("hello"); + proc->in.close(); + REQUIRE(proc->wait() == 0); + REQUIRE(proc->out.getline() == "hello"); + REQUIRE(!proc->out.getline()); + } + // several lines, and writes made through stream() rather than putline(). + { + auto proc = util::run({"cat"}); + REQUIRE(proc); + proc->in.putline("one"); + proc->in.stream() << "two\n"; + proc->in.close(); + REQUIRE(proc->wait() == 0); + REQUIRE(proc->out.getline() == "one"); + REQUIRE(proc->out.getline() == "two"); + REQUIRE(!proc->out.getline()); + } + // close() is idempotent: the destructor closes too, and callers that + // close early must not double-close the file descriptor. + { + auto proc = util::run({"cat"}); + REQUIRE(proc); + proc->in.putline("once"); + proc->in.close(); + proc->in.close(); + REQUIRE(proc->wait() == 0); + REQUIRE(proc->out.getline() == "once"); + } + // a child that consumes stdin and ignores it still terminates. + { + auto proc = util::run({"wc", "-c"}); + REQUIRE(proc); + proc->in.stream() << "12345"; + proc->in.close(); + REQUIRE(proc->wait() == 0); + auto line = proc->out.getline(); + REQUIRE(line); + REQUIRE_THAT(*line, matchers::ContainsSubstring("5")); + } +} + TEST_CASE("stdout", "[subprocess]") { { auto proc = util::run({"echo", "hello world"}); From 87381185aff1f16e3ff54d31813e1c5cb820763c Mon Sep 17 00:00:00 2001 From: bcumming Date: Fri, 10 Jul 2026 17:35:50 +0200 Subject: [PATCH 29/29] tidy up progress bars; add progress bar for tracking image validation when pushing/adding --- src/cli/add_remove.cpp | 20 ++++++++- src/cli/pull.cpp | 36 ++++----------- src/cli/push.cpp | 67 ++++++++++++++------------- src/cli/util.cpp | 66 +++++++++++++++++++++++++-- src/cli/util.h | 50 ++++++++++++++++++++- src/oci/push.cpp | 15 +++++-- src/oci/push.h | 3 ++ src/util/sha256.cpp | 14 +++++- src/util/sha256.h | 7 ++- test/integration/registry.bats | 5 +++ test/unit/oci_registry.cpp | 37 +++++++++++++++ test/unit/sha256.cpp | 82 ++++++++++++++++++++++++++++++++++ 12 files changed, 331 insertions(+), 71 deletions(-) diff --git a/src/cli/add_remove.cpp b/src/cli/add_remove.cpp index 51c38d21..892a6417 100644 --- a/src/cli/add_remove.cpp +++ b/src/cli/add_remove.cpp @@ -1,5 +1,6 @@ // vim: ts=4 sts=4 sw=4 et +#include #include #include @@ -103,7 +104,24 @@ int image_add(const image_add_args& args, const global_settings& settings) { sqfs = squashfs_image{env->sqfs_path, env->meta_path, fmt::format("{}", env->record->sha)}; } else { - sqfs = uenv::validate_squashfs_image(env->sqfs_path); + // hashing the image reads the whole file, so show progress for it. The + // bar is created on the first callback, once the size is known. + std::unique_ptr prepare_bar; + sqfs = uenv::validate_squashfs_image( + env->sqfs_path, + [&prepare_bar, &env](std::uint64_t done, std::uint64_t total) { + if (!prepare_bar) { + prepare_bar = uenv::make_transfer_bar( + total, fmt::format("preparing {}", + env->sqfs_path.filename().string())); + } + prepare_bar->update(done); + }); + // stop the bar before anything else is printed, so that an error + // message does not land on the bar's line. + if (prepare_bar) { + prepare_bar->finish(); + } if (!sqfs) { term::error("invalid source {}: {}", args.source, sqfs.error()); return 1; diff --git a/src/cli/pull.cpp b/src/cli/pull.cpp index 46d52b7a..faa606db 100644 --- a/src/cli/pull.cpp +++ b/src/cli/pull.cpp @@ -1,11 +1,9 @@ // vim: ts=4 sts=4 sw=4 et -#include #include #include #include -#include #include #include #include @@ -244,30 +242,12 @@ int image_pull(const image_pull_args& args, const global_settings& settings) { } if (pull_sqfs) { - namespace bk = barkeep; - // atomic: written by curl's progress callback on the main - // thread, read concurrently by barkeep's display thread. - std::atomic downloaded_mb{0u}; - // round up so total_mb is never zero - std::size_t total_mb{(record.size_byte + (1024 * 1024 - 1)) / - (1024 * 1024)}; - auto bar = bk::ProgressBar( - &downloaded_mb, - { - .total = total_mb, - .message = - fmt::format("pulling {}", record.id.string()), - .speed = 0.1, - .speed_unit = "MB/s", - .style = color::use_color() - ? bk::ProgressBarStyle::Rich - : bk::ProgressBarStyle::Bars, - .no_tty = !isatty(fileno(stdout)), - }); + auto bar = uenv::make_transfer_bar( + record.size_byte, + fmt::format("pulling {}", record.id.string())); - auto progress = [&downloaded_mb](std::uint64_t now, - std::uint64_t) { - downloaded_mb = now / (1024 * 1024); + auto progress = [&bar](std::uint64_t now, std::uint64_t) { + bar->update(now); }; // util::signal_raised() consumes (resets) the flag, so it must // be checked exactly once. latch the result here in the abort @@ -283,10 +263,10 @@ int image_pull(const image_pull_args& args, const global_settings& settings) { return aborted; }); if (!aborted) { - // ensure that the progress bar shows %100 on completion - downloaded_mb = total_mb; + bar->finish(); + } else { + bar->stop(); } - bar->done(); if (!result) { // a Ctrl-C during the download aborts the transfer; surface diff --git a/src/cli/push.cpp b/src/cli/push.cpp index 3ced7d05..3f92a738 100644 --- a/src/cli/push.cpp +++ b/src/cli/push.cpp @@ -1,7 +1,7 @@ // vim: ts=4 sts=4 sw=4 et -#include #include +#include #include #include @@ -125,8 +125,27 @@ int image_push([[maybe_unused]] const image_push_args& args, term::msg("the destination already exists and will be overwritten"); } - // validate the source squashfs file - auto sqfs = uenv::validate_squashfs_image(args.source); + // validate the source squashfs file. This hashes the whole image, which + // takes minutes for a multi-GB uenv, so show a progress bar for it. The bar + // is created on the first callback, once the image size is known. + std::unique_ptr prepare_bar; + auto sqfs = uenv::validate_squashfs_image( + args.source, + [&prepare_bar, &args](std::uint64_t done, std::uint64_t total) { + if (!prepare_bar) { + prepare_bar = uenv::make_transfer_bar( + total, fmt::format("preparing {}", + std::filesystem::path{args.source} + .filename() + .string())); + } + prepare_bar->update(done); + }); + // stop the bar before anything else is printed, so that an error message + // does not land on the bar's line. + if (prepare_bar) { + prepare_bar->finish(); + } if (!sqfs) { term::error("invalid squashfs file {}: {}", args.source, sqfs.error()); return 1; @@ -158,39 +177,27 @@ int image_push([[maybe_unused]] const image_push_args& args, // interrupt leaves nothing referencing a partial blob. std::optional image_digest; { - // atomic: written by curl's progress callback on the main thread, read - // concurrently by barkeep's display thread. - std::atomic uploaded_mb{0u}; std::error_code ec; auto size_byte = std::filesystem::file_size(sqfs->sqfs, ec); - // round up so total_mb is never zero - std::size_t total_mb{ - ec ? 0u : (size_byte + (1024 * 1024 - 1)) / (1024 * 1024)}; - auto bar = bk::ProgressBar( - &uploaded_mb, - { - .total = total_mb, - .message = fmt::format("pushing {} to registry", - sqfs->sqfs.filename().string()), - .speed = 0.1, - .speed_unit = "MB/s", - .style = color::use_color() ? bk::ProgressBarStyle::Rich - : bk::ProgressBarStyle::Bars, - .no_tty = !isatty(fileno(stdout)), - }); + auto bar = uenv::make_transfer_bar( + ec ? 0u : size_byte, fmt::format("pushing {} to registry", + sqfs->sqfs.filename().string())); - auto progress = [&uploaded_mb](std::uint64_t now, std::uint64_t) { - uploaded_mb = now / (1024 * 1024); + auto progress = [&bar](std::uint64_t now, std::uint64_t) { + bar->update(now); }; - auto push_result = oci::push_squashfs( - *client, sqfs->sqfs, oci::reference::tag(*L.tag), progress); + // the image was hashed by validate_squashfs_image; pass that digest in + // rather than reading the whole file a second time. + auto push_result = + oci::push_squashfs(*client, sqfs->sqfs, oci::reference::tag(*L.tag), + oci::digest::sha256(sqfs->hash), progress); + // fill the bar on success, which also covers the case where the + // registry already had the blob and no upload happened. if (push_result) { - // ensure the progress bar shows 100% on completion (also covers the - // case where the registry already had the blob and no upload - // happened). - uploaded_mb = total_mb; + bar->finish(); + } else { + bar->stop(); } - bar->done(); if (!push_result) { term::error("unable to push uenv.\n{}", push_result.error()); return 1; diff --git a/src/cli/util.cpp b/src/cli/util.cpp index 9b62b084..18a001af 100644 --- a/src/cli/util.cpp +++ b/src/cli/util.cpp @@ -1,10 +1,14 @@ +#include #include +#include + #include #include #include #include +#include #include #include #include @@ -105,8 +109,53 @@ squashfs_mount_args(const envvars::state& calling_environment, return commands; } -util::expected -validate_squashfs_image(const std::string& path) { +namespace { +// bytes -> whole megabytes, rounded up. +std::size_t to_mb(std::uint64_t bytes) { + constexpr std::uint64_t mb = 1024 * 1024; + return static_cast((bytes + mb - 1) / mb); +} +} // namespace + +transfer_bar::transfer_bar(std::uint64_t total_bytes, std::string message) + // never zero: barkeep divides by the total to render the bar. + : total_mb_{std::max(1u, to_mb(total_bytes))} { + namespace bk = barkeep; + bar_ = bk::ProgressBar(&transferred_mb_, + { + .total = total_mb_, + .message = std::move(message), + .speed = 0.1, + .speed_unit = "MB/s", + .style = color::use_color() + ? bk::ProgressBarStyle::Rich + : bk::ProgressBarStyle::Bars, + .no_tty = !isatty(fileno(stdout)), + }); +} + +void transfer_bar::update(std::uint64_t bytes) { + transferred_mb_.store(to_mb(bytes), std::memory_order_relaxed); +} + +void transfer_bar::finish() { + transferred_mb_.store(total_mb_, std::memory_order_relaxed); + bar_->done(); +} + +void transfer_bar::stop() { + bar_->done(); +} + +std::unique_ptr make_transfer_bar(std::uint64_t total_bytes, + std::string message) { + return std::make_unique(total_bytes, std::move(message)); +} + +util::expected validate_squashfs_image( + const std::string& path, + std::function + hash_progress) { namespace fs = std::filesystem; squashfs_image img{}; @@ -130,7 +179,18 @@ validate_squashfs_image(const std::string& path) { spdlog::info("no meta data in {}", img.sqfs); } - auto hash = util::sha256_file(img.sqfs); + std::error_code ec; + const auto total = fs::file_size(img.sqfs, ec); + // report the total up front, so a caller's bar appears before the first + // chunk is hashed rather than after it. + if (hash_progress) { + hash_progress(0u, ec ? 0u : total); + } + auto hash = util::sha256_file(img.sqfs, [&](std::uint64_t hashed) { + if (hash_progress) { + hash_progress(hashed, ec ? 0u : total); + } + }); if (!hash) { spdlog::error("{}", hash.error()); return util::unexpected{fmt::format( diff --git a/src/cli/util.h b/src/cli/util.h index a9189003..d02a3fa7 100644 --- a/src/cli/util.h +++ b/src/cli/util.h @@ -1,10 +1,15 @@ #pragma once +#include +#include #include +#include +#include #include #include #include +#include #include #include @@ -25,6 +30,41 @@ resolve_registry_credentials(const envvars::state& env, std::optional username, std::optional token); +// A progress bar over a byte transfer: hashing, uploading or downloading a +// squashfs image. Progress is reported and displayed in whole megabytes. +// +// barkeep polls the counter through a pointer, from its own display thread, so +// this owns the counter and must never be copied or moved - doing so would +// leave that thread reading a dangling address. Construct one with +// make_transfer_bar, which keeps it at a stable address on the heap. +class transfer_bar { + public: + transfer_bar(std::uint64_t total_bytes, std::string message); + + transfer_bar(const transfer_bar&) = delete; + transfer_bar& operator=(const transfer_bar&) = delete; + transfer_bar(transfer_bar&&) = delete; + transfer_bar& operator=(transfer_bar&&) = delete; + + // report the cumulative number of bytes transferred so far. + void update(std::uint64_t bytes); + + // fill the bar and stop it. Also covers a transfer that was skipped + // entirely, e.g. a blob the registry already had. + void finish(); + + // stop the bar where it is, for a transfer that was aborted. + void stop(); + + private: + std::atomic transferred_mb_{0u}; + std::size_t total_mb_; + std::shared_ptr bar_; +}; + +std::unique_ptr make_transfer_bar(std::uint64_t total_bytes, + std::string message); + struct squashfs_image { // the absolute path of the squashfs file std::filesystem::path sqfs; @@ -36,8 +76,14 @@ struct squashfs_image { std::string hash; }; -util::expected -validate_squashfs_image(const std::string& path); +// Hashing the squashfs reads the whole file, which takes minutes for a +// multi-GB image. `hash_progress`, when set, is called with the cumulative +// bytes hashed and the total, so a caller can drive a progress bar. It is +// called once with (0, total) before hashing begins. +util::expected validate_squashfs_image( + const std::string& path, + std::function hash_progress = + {}); std::vector squashfs_mount_args(const envvars::state& calling_environment, diff --git a/src/oci/push.cpp b/src/oci/push.cpp index 54904ec6..a4f06227 100644 --- a/src/oci/push.cpp +++ b/src/oci/push.cpp @@ -255,10 +255,16 @@ void maintain_referrers_tag(client& c, const digest& subject, util::expected push_squashfs(client& c, const fs::path& squashfs, const reference& ref, + std::optional layer_digest, std::function progress) { - auto layer_digest = digest_of_file(squashfs); + // hashing a multi-GB squashfs is expensive, so a caller that already has + // the digest passes it in rather than paying for a second pass. if (!layer_digest) { - return util::unexpected{layer_digest.error()}; + auto computed = digest_of_file(squashfs); + if (!computed) { + return util::unexpected{computed.error()}; + } + layer_digest = computed.value(); } std::error_code ec; auto size = fs::file_size(squashfs, ec); @@ -271,7 +277,8 @@ push_squashfs(client& c, const fs::path& squashfs, const reference& ref, layer_digest->string(), size, ref.string()); // upload the squashfs blob (streamed) and the empty config. - if (auto ok = c.put_blob(*layer_digest, squashfs, std::move(progress)); + if (auto ok = + c.put_blob(layer_digest.value(), squashfs, std::move(progress)); !ok) { return util::unexpected{ok.error().message}; } @@ -284,7 +291,7 @@ push_squashfs(client& c, const fs::path& squashfs, const reference& ref, m.annotations[std::string{annotation_created}] = rfc3339_now(); m.layers.push_back(manifest_layer{ .media_type = std::string{media_type_layer_tar}, - .digest = *layer_digest, + .digest = layer_digest.value(), .size = size, .annotations = {{std::string{annotation_title}, "store.squashfs"}}}); diff --git a/src/oci/push.h b/src/oci/push.h index 6ab6bb0d..e1278d53 100644 --- a/src/oci/push.h +++ b/src/oci/push.h @@ -19,9 +19,12 @@ namespace oci { // blob and the empty config, builds an oras-compatible image manifest with the // layer titled "store.squashfs", and PUTs it under `ref` (a tag). Returns the // manifest digest — the canonical image id. +// `layer_digest` is the sha256 of `squashfs`; supply it when it has already +// been computed, else it is calculated here, which reads the whole file. util::expected push_squashfs(client& c, const std::filesystem::path& squashfs, const reference& ref, + std::optional layer_digest = std::nullopt, std::function progress = {}); // Attach typed side-data to an existing image as an OCI referrer (the native diff --git a/src/util/sha256.cpp b/src/util/sha256.cpp index dbadb731..27e823ab 100644 --- a/src/util/sha256.cpp +++ b/src/util/sha256.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include @@ -216,7 +217,8 @@ sha256_digest sha256_string(std::string_view text) { } util::expected -sha256_file(const std::filesystem::path& path) { +sha256_file(const std::filesystem::path& path, + std::function progress) { std::FILE* f = std::fopen(path.c_str(), "rb"); if (f == nullptr) { return util::unexpected{fmt::format("unable to open {} for reading: {}", @@ -226,10 +228,18 @@ sha256_file(const std::filesystem::path& path) { sha256_state s; sha256_init(s); - std::array buf; + // one chunk per megabyte: fine-grained enough to drive a progress bar + // reported in MB, without calling back hundreds of thousands of times on a + // multi-GB file. Heap-allocated: too large to sit on a thread stack. + std::vector buf(1 << 20); + std::uint64_t hashed = 0; std::size_t n; while ((n = std::fread(buf.data(), 1, buf.size(), f)) > 0) { sha256_update(s, std::span{buf.data(), n}); + hashed += n; + if (progress) { + progress(hashed); + } } if (std::ferror(f)) { diff --git a/src/util/sha256.h b/src/util/sha256.h index 80da53f0..f8432f56 100644 --- a/src/util/sha256.h +++ b/src/util/sha256.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -53,7 +54,11 @@ sha256_digest sha256_string(std::string_view text); // digest a file by streaming it in fixed-size chunks, so large files never // need to be held in memory. returns an error message if the file is // unreadable. +// `progress`, when set, is called once per chunk with the cumulative number of +// bytes hashed so far, so that callers can drive a progress bar over a +// multi-GB file. It is called from this thread, and the hash is not abortable. util::expected -sha256_file(const std::filesystem::path& path); +sha256_file(const std::filesystem::path& path, + std::function progress = {}); } // namespace util diff --git a/test/integration/registry.bats b/test/integration/registry.bats index e0c04700..2d2f2d48 100644 --- a/test/integration/registry.bats +++ b/test/integration/registry.bats @@ -82,6 +82,11 @@ function teardown() { log "${output}" [ "${status}" -eq 0 ] + # the image is hashed, then uploaded: each phase reports its own progress + # bar. Even with no tty, barkeep prints the bar's first and last line. + assert_output --partial "preparing" + assert_output --partial "pushing" + # register a listing record so pull can resolve the label local repo="${REG_PREFIX}/deploy/arapiles/zen3/app/1.0" local digest diff --git a/test/unit/oci_registry.cpp b/test/unit/oci_registry.cpp index a59ad16c..82d3c0d5 100644 --- a/test/unit/oci_registry.cpp +++ b/test/unit/oci_registry.cpp @@ -331,3 +331,40 @@ TEST_CASE("oci registry copy preserves digest", "[registry]") { REQUIRE(tag_refs->size() == 1); REQUIRE(tag_refs->front().artifact_type == "uenv/meta"); } + +TEST_CASE("oci registry push_squashfs with a precomputed digest", + "[registry]") { + const auto base = registry_base(); + if (base.empty()) { + SKIP("no zot binary available for the registry tests"); + } + + auto dir = util::make_temp_dir().value(); + const auto sqfs = dir / "store.squashfs"; + const std::string payload = "squashfs-bytes-precomputed-digest"; + write_file(sqfs, payload); + + auto c = oci::client::create(base, "test/precomputed/1.0"); + REQUIRE(c.has_value()); + + // supplying the layer digest spares push_squashfs a full read of the file. + const auto layer = oci::digest::from_sha256(util::sha256_string(payload)); + auto pushed = + oci::push_squashfs(*c, sqfs, oci::reference::tag("v1"), layer); + REQUIRE(pushed.has_value()); + + // the manifest records the digest we supplied, and the layer is addressable + // under it. (The manifest digest itself is not compared against a + // hash-it-yourself push: push_squashfs stamps a wall-clock `created` + // annotation, so two pushes need not produce identical manifests.) + auto resp = c->get_manifest(oci::reference::tag("v1")); + REQUIRE(resp.has_value()); + auto image = oci::parse_manifest(resp->body); + REQUIRE(image.has_value()); + REQUIRE(image->layers.size() == 1); + REQUIRE(image->layers[0].digest == layer); + + const auto blob = dir / "pulled.squashfs"; + REQUIRE(c->get_blob_to_file(layer, blob).has_value()); + REQUIRE(read_file(blob) == payload); +} diff --git a/test/unit/sha256.cpp b/test/unit/sha256.cpp index 4396497c..f241c65b 100644 --- a/test/unit/sha256.cpp +++ b/test/unit/sha256.cpp @@ -1,10 +1,15 @@ +#include #include +#include +#include #include #include #include +#include #include +#include #include namespace { @@ -93,3 +98,80 @@ TEST_CASE("sha256 update splits do not change the digest", "[sha256]") { REQUIRE(util::sha256_final(s).hex() == whole); } } + +namespace { +// write `content` to a fresh file in a temporary directory, and return its +// path. +std::filesystem::path scratch_file(std::string_view content) { + auto dir = util::make_temp_dir().value(); + auto path = dir / "payload.bin"; + std::ofstream f{path, std::ios::binary}; + f.write(content.data(), static_cast(content.size())); + f.close(); + return path; +} +} // namespace + +TEST_CASE("sha256_file", "[sha256]") { + // sha256_file streams the file in 1 MiB chunks; size the cases around that. + constexpr std::size_t chunk = 1u << 20; + + SECTION("matches the one-shot digest") { + const std::string content(3 * chunk + 12345, 'a'); + auto d = util::sha256_file(scratch_file(content)); + REQUIRE(d); + REQUIRE(d->hex() == util::sha256_string(content).hex()); + } + + SECTION("an empty file digests the empty string") { + auto d = util::sha256_file(scratch_file("")); + REQUIRE(d); + REQUIRE(d->hex() == util::sha256_string("").hex()); + } + + SECTION("a missing file is an error") { + REQUIRE_FALSE(util::sha256_file("/no/such/file-xyz123")); + } + + // the progress callback reports cumulative bytes hashed. + SECTION("progress is monotonic and ends at the file size") { + const std::string content(3 * chunk + 12345, 'b'); + auto path = scratch_file(content); + + std::vector seen; + auto d = util::sha256_file( + path, [&seen](std::uint64_t hashed) { seen.push_back(hashed); }); + REQUIRE(d); + + // instrumenting the read must not corrupt the digest. + REQUIRE(d->hex() == util::sha256_string(content).hex()); + + REQUIRE_FALSE(seen.empty()); + REQUIRE(std::is_sorted(seen.begin(), seen.end())); + REQUIRE(std::adjacent_find(seen.begin(), seen.end()) == seen.end()); + REQUIRE(seen.back() == content.size()); + // 3 full chunks plus a partial one. + REQUIRE(seen.size() == 4); + } + + SECTION("a file below the chunk size reports once") { + const std::string content(1024, 'c'); + std::vector seen; + auto d = util::sha256_file( + scratch_file(content), + [&seen](std::uint64_t hashed) { seen.push_back(hashed); }); + REQUIRE(d); + REQUIRE(seen.size() == 1); + REQUIRE(seen.front() == content.size()); + } + + SECTION("an empty file never reports progress") { + std::vector seen; + auto d = + util::sha256_file(scratch_file(""), [&seen](std::uint64_t hashed) { + seen.push_back(hashed); + }); + REQUIRE(d); + REQUIRE(seen.empty()); + } +}