From 11ff8ab601df41ddf09dfdfc8a530deb00dd3f62 Mon Sep 17 00:00:00 2001 From: Tomas David Date: Thu, 30 Jul 2026 15:14:04 +0200 Subject: [PATCH 1/5] ART-21809: Add doozer verify-image-consistency command Verify that every image in the release payload is present in the shipment MR or has already been released in the Red Hat catalog. Compares payload images (oc adm release info --pullspecs) against shipment components from GitLab MR YAML files. RHCOS images are skipped. Images matched by digest, list digest, or VCS reference, with Red Hat Catalog API as fallback. Co-Authored-By: Claude Opus 4.6 rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED --- doozer/doozerlib/cli/__main__.py | 1 + .../doozerlib/cli/verify_image_consistency.py | 318 ++++++++++++++++++ .../cli/test_verify_image_consistency.py | 262 +++++++++++++++ 3 files changed, 581 insertions(+) create mode 100644 doozer/doozerlib/cli/verify_image_consistency.py create mode 100644 doozer/tests/cli/test_verify_image_consistency.py diff --git a/doozer/doozerlib/cli/__main__.py b/doozer/doozerlib/cli/__main__.py index 033b70d485..5c976587bc 100644 --- a/doozer/doozerlib/cli/__main__.py +++ b/doozer/doozerlib/cli/__main__.py @@ -73,6 +73,7 @@ from doozerlib.cli.scan_osh import scan_osh from doozerlib.cli.scan_sources import config_scan_source_changes from doozerlib.cli.scan_sources_konflux import config_scan_source_changes_konflux +from doozerlib.cli.verify_image_consistency import verify_image_consistency_cli from doozerlib.exceptions import DoozerFatalError from doozerlib.util import analyze_debug_timing diff --git a/doozer/doozerlib/cli/verify_image_consistency.py b/doozer/doozerlib/cli/verify_image_consistency.py new file mode 100644 index 0000000000..2602e625f9 --- /dev/null +++ b/doozer/doozerlib/cli/verify_image_consistency.py @@ -0,0 +1,318 @@ +import asyncio +import json +import logging +import re +from dataclasses import dataclass, field +from typing import Optional + +import aiohttp +import click +import yaml +from artcommonlib import exectools +from artcommonlib.gitlab import GitLabClient +from artcommonlib.oc_image_info import oc_image_info__cached_async + +from doozerlib.cli import cli, click_coroutine, pass_runtime + +LOGGER = logging.getLogger(__name__) + +SKIPPED_IMAGES_PATTERNS = [ + re.compile(r"machine-os-content"), + re.compile(r"rhel-coreos(?:-\d+)?"), + re.compile(r"rhel-coreos(?:-\d+)?-extensions"), +] + +CATALOG_API_URL = "https://catalog.redhat.com/api/containers/v1/images" + + +@dataclass +class ImageCheckResult: + name: str + pullspec: str + found_in: Optional[str] = None + match_details: Optional[str] = None + + @property + def passed(self) -> bool: + return self.found_in is not None + + +@dataclass +class VerifyImageConsistencyResult: + payload_url: str + shipment_mr_url: str + payload_version: Optional[str] = None + shipment_version: Optional[str] = None + payload_image_count: int = 0 + shipment_component_count: int = 0 + skipped_images: list[str] = field(default_factory=list) + results: list[ImageCheckResult] = field(default_factory=list) + + @property + def passed(self) -> bool: + return all(r.passed for r in self.results) + + @property + def failed_images(self) -> list[ImageCheckResult]: + return [r for r in self.results if not r.passed] + + +def _is_skipped_image(name: str) -> bool: + return any(p.fullmatch(name) for p in SKIPPED_IMAGES_PATTERNS) + + +@dataclass +class ImageIdentifiers: + pullspec: str + digest: str = "" + list_digest: str = "" + vcs_ref: str = "" + name: str = "" + + +def identifiers_match(a: ImageIdentifiers, b: ImageIdentifiers) -> bool: + if a.list_digest and a.list_digest == b.list_digest: + return True + if a.digest and a.digest == b.digest: + return True + if a.vcs_ref and a.vcs_ref == b.vcs_ref: + return True + return False + + +async def fetch_payload_images(payload_url: str) -> tuple[list[tuple[str, str]], str]: + LOGGER.info("Fetching payload data from %s", payload_url) + cmd = ["oc", "adm", "release", "info", "--pullspecs", payload_url, "-o", "json"] + rc, stdout, stderr = await exectools.cmd_gather_async(cmd, check=False) + if rc: + raise RuntimeError(f"oc adm release info failed (rc={rc}): {stderr.strip()}") + + data = json.loads(stdout) + version = data.get("metadata", {}).get("version", "") + tags = data.get("references", {}).get("spec", {}).get("tags", []) + + images = [] + for tag in tags: + name = tag.get("name", "") + pullspec = tag.get("from", {}).get("name", "") + if name and pullspec: + images.append((name, pullspec)) + + return images, version + + +def fetch_shipment_components(mr_url: str) -> tuple[list[tuple[str, str]], str]: + gl = GitLabClient.from_url(mr_url) + mr = gl.get_mr_from_url(mr_url) + source_project = gl.get_project(mr.source_project_id) + + title = mr.title or "" + match = re.search(r"Shipment for (\d+\.\d+\.\d+(?:-\S+)?)", title, re.IGNORECASE) + version = match.group(1) if match else "" + + diff_info = mr.diffs.list(all=True)[0] + diff = mr.diffs.get(diff_info.id) + + components: list[tuple[str, str]] = [] + for file_diff in diff.diffs: + file_path = file_diff.get("new_path") or file_diff.get("old_path") + if not file_path or not file_path.endswith((".yaml", ".yml")): + continue + + try: + file_content = source_project.files.get(file_path, mr.source_branch) + content = file_content.decode().decode("utf-8") + data = yaml.safe_load(content) + + shipment = data.get("shipment") or {} + snapshot = shipment.get("snapshot") or {} + spec = snapshot.get("spec") or {} + for comp in spec.get("components") or []: + name = comp.get("name", "") + pullspec = comp.get("containerImage", "") + if pullspec: + components.append((name, pullspec)) + except Exception: + LOGGER.warning("Failed to process shipment file %s in MR %s", file_path, mr_url, exc_info=True) + continue + + return components, version + + +async def fetch_image_identifiers(pullspec: str) -> ImageIdentifiers: + try: + stdout = await oc_image_info__cached_async(pullspec, "--filter-by-os=linux/amd64", "--insecure=true") + data = json.loads(stdout) + except Exception: + LOGGER.warning("Failed to fetch image metadata for %s", pullspec, exc_info=True) + return ImageIdentifiers(pullspec=pullspec) + + labels = data.get("config", {}).get("config", {}).get("Labels", {}) + return ImageIdentifiers( + pullspec=pullspec, + digest=data.get("digest", ""), + list_digest=data.get("listDigest", ""), + vcs_ref=labels.get("vcs-ref", ""), + name=labels.get("name", ""), + ) + + +async def check_catalog(digest: str) -> bool: + if not digest: + return False + + url = f"{CATALOG_API_URL}?filter=image_id=={digest}" + try: + async with aiohttp.ClientSession() as session: + async with session.get(url) as resp: + if resp.status != 200: + LOGGER.warning("Red Hat Catalog API returned status %s", resp.status) + return False + data = await resp.json() + return data.get("total", 0) > 0 + except Exception: + LOGGER.warning("Failed to query Red Hat Catalog API for digest %s", digest, exc_info=True) + return False + + +async def verify_image_consistency(payload_url: str, shipment_mr_url: str) -> VerifyImageConsistencyResult: + payload_images_task = fetch_payload_images(payload_url) + shipment_task = asyncio.to_thread(fetch_shipment_components, shipment_mr_url) + + payload_images, payload_version = await payload_images_task + shipment_components, shipment_version = await shipment_task + + result = VerifyImageConsistencyResult( + payload_url=payload_url, + shipment_mr_url=shipment_mr_url, + payload_version=payload_version, + shipment_version=shipment_version, + payload_image_count=len(payload_images), + shipment_component_count=len(shipment_components), + ) + + images_to_check = [] + for name, pullspec in payload_images: + if _is_skipped_image(name): + result.skipped_images.append(name) + LOGGER.info("Skipping RHCOS image: %s", name) + continue + images_to_check.append((name, pullspec)) + + LOGGER.info( + "Checking %d payload images against %d shipment components (%d skipped)", + len(images_to_check), + len(shipment_components), + len(result.skipped_images), + ) + + all_pullspecs = set() + for _, pullspec in images_to_check: + all_pullspecs.add(pullspec) + for _, pullspec in shipment_components: + all_pullspecs.add(pullspec) + + identifiers_tasks = {ps: fetch_image_identifiers(ps) for ps in all_pullspecs} + identifiers: dict[str, ImageIdentifiers] = {} + for ps, task in identifiers_tasks.items(): + identifiers[ps] = await task + + shipment_identifiers = [identifiers[ps] for _, ps in shipment_components if ps in identifiers] + + for name, pullspec in images_to_check: + payload_id = identifiers.get(pullspec, ImageIdentifiers(pullspec=pullspec)) + check = ImageCheckResult(name=name, pullspec=pullspec) + + for ship_id in shipment_identifiers: + if identifiers_match(payload_id, ship_id): + check.found_in = "shipment" + check.match_details = ship_id.pullspec + break + + if not check.found_in: + if await check_catalog(payload_id.digest): + check.found_in = "catalog" + + if not check.found_in: + LOGGER.error("Image %s (%s) not found in shipment or catalog", name, pullspec) + + result.results.append(check) + + return result + + +def render_result(result: VerifyImageConsistencyResult, output: str) -> str: + if output == "json": + return json.dumps( + { + "payload_url": result.payload_url, + "shipment_mr_url": result.shipment_mr_url, + "payload_version": result.payload_version, + "shipment_version": result.shipment_version, + "payload_image_count": result.payload_image_count, + "shipment_component_count": result.shipment_component_count, + "skipped_images": result.skipped_images, + "passed": result.passed, + "failed_images": [{"name": r.name, "pullspec": r.pullspec} for r in result.failed_images], + "results": [ + { + "name": r.name, + "pullspec": r.pullspec, + "passed": r.passed, + "found_in": r.found_in, + } + for r in result.results + ], + }, + indent=2, + ) + + lines = [ + f"Image consistency check for payload {result.payload_version or result.payload_url}", + f"Shipment MR: {result.shipment_mr_url}", + f"Payload images: {result.payload_image_count} ({len(result.skipped_images)} RHCOS skipped)", + f"Shipment components: {result.shipment_component_count}", + ] + + if result.failed_images: + lines.append("") + lines.append("IMAGES NOT FOUND IN SHIPMENT OR CATALOG:") + for r in result.failed_images: + lines.append(f" - {r.name}: {r.pullspec}") + + lines.append("") + overall = "PASS" if result.passed else "FAIL" + lines.append(f"Overall: {overall} ({len(result.results) - len(result.failed_images)}/{len(result.results)} passed)") + return "\n".join(lines) + + +@cli.command("verify-image-consistency", short_help="Verify payload images match shipment MR components") +@click.option( + "--payload-url", + required=True, + help="Release payload pullspec, e.g. quay.io/openshift-release-dev/ocp-release:4.20.1-x86_64", +) +@click.option("--shipment-mr-url", required=True, help="Shipment GitLab MR URL") +@click.option( + "-o", + "--output", + type=click.Choice(["text", "json"]), + default="text", + show_default=True, + help="Output format.", +) +@pass_runtime +@click_coroutine +async def verify_image_consistency_cli(runtime, payload_url, shipment_mr_url, output): + """Verify that every image in the release payload is present in the + shipment MR or has already been released in the Red Hat catalog. + + Compares payload images (from oc adm release info) against shipment + components (from the GitLab MR YAML files). RHCOS images are skipped. + Images are matched by digest, list digest, or VCS reference. + """ + runtime.initialize(no_group=True) + result = await verify_image_consistency(payload_url=payload_url, shipment_mr_url=shipment_mr_url) + click.echo(render_result(result, output)) + if not result.passed: + raise SystemExit(1) diff --git a/doozer/tests/cli/test_verify_image_consistency.py b/doozer/tests/cli/test_verify_image_consistency.py new file mode 100644 index 0000000000..7e0b08cafa --- /dev/null +++ b/doozer/tests/cli/test_verify_image_consistency.py @@ -0,0 +1,262 @@ +import json +from unittest import IsolatedAsyncioTestCase +from unittest.mock import patch + +from doozerlib.cli.verify_image_consistency import ( + ImageCheckResult, + ImageIdentifiers, + VerifyImageConsistencyResult, + _is_skipped_image, + fetch_image_identifiers, + fetch_payload_images, + identifiers_match, + render_result, + verify_image_consistency, +) + + +class TestIsSkippedImage(IsolatedAsyncioTestCase): + def test_machine_os_content(self): + self.assertTrue(_is_skipped_image("machine-os-content")) + + def test_rhel_coreos(self): + self.assertTrue(_is_skipped_image("rhel-coreos")) + + def test_rhel_coreos_9(self): + self.assertTrue(_is_skipped_image("rhel-coreos-9")) + + def test_rhel_coreos_extensions(self): + self.assertTrue(_is_skipped_image("rhel-coreos-extensions")) + + def test_rhel_coreos_9_extensions(self): + self.assertTrue(_is_skipped_image("rhel-coreos-9-extensions")) + + def test_regular_image(self): + self.assertFalse(_is_skipped_image("ose-cli")) + + def test_partial_match_not_skipped(self): + self.assertFalse(_is_skipped_image("rhel-coreos-extra")) + + +class TestIdentifiersMatch(IsolatedAsyncioTestCase): + def test_match_by_list_digest(self): + a = ImageIdentifiers(pullspec="a", list_digest="sha256:abc") + b = ImageIdentifiers(pullspec="b", list_digest="sha256:abc") + self.assertTrue(identifiers_match(a, b)) + + def test_match_by_digest(self): + a = ImageIdentifiers(pullspec="a", digest="sha256:def") + b = ImageIdentifiers(pullspec="b", digest="sha256:def") + self.assertTrue(identifiers_match(a, b)) + + def test_match_by_vcs_ref(self): + a = ImageIdentifiers(pullspec="a", vcs_ref="abc123") + b = ImageIdentifiers(pullspec="b", vcs_ref="abc123") + self.assertTrue(identifiers_match(a, b)) + + def test_no_match(self): + a = ImageIdentifiers(pullspec="a", digest="sha256:aaa", vcs_ref="111") + b = ImageIdentifiers(pullspec="b", digest="sha256:bbb", vcs_ref="222") + self.assertFalse(identifiers_match(a, b)) + + def test_empty_fields_no_match(self): + a = ImageIdentifiers(pullspec="a") + b = ImageIdentifiers(pullspec="b") + self.assertFalse(identifiers_match(a, b)) + + def test_list_digest_takes_priority(self): + a = ImageIdentifiers(pullspec="a", list_digest="sha256:same", digest="sha256:diff_a") + b = ImageIdentifiers(pullspec="b", list_digest="sha256:same", digest="sha256:diff_b") + self.assertTrue(identifiers_match(a, b)) + + +class TestFetchPayloadImages(IsolatedAsyncioTestCase): + @patch("doozerlib.cli.verify_image_consistency.exectools.cmd_gather_async") + async def test_returns_images_and_version(self, mock_cmd): + payload_data = { + "metadata": {"version": "4.20.1"}, + "references": { + "spec": { + "tags": [ + {"name": "ose-cli", "from": {"name": "quay.io/ocp/cli@sha256:abc"}}, + {"name": "machine-os-content", "from": {"name": "quay.io/ocp/rhcos@sha256:def"}}, + ] + } + }, + } + mock_cmd.return_value = (0, json.dumps(payload_data), "") + + images, version = await fetch_payload_images("quay.io/ocp:4.20.1") + self.assertEqual(version, "4.20.1") + self.assertEqual(len(images), 2) + self.assertEqual(images[0], ("ose-cli", "quay.io/ocp/cli@sha256:abc")) + + @patch("doozerlib.cli.verify_image_consistency.exectools.cmd_gather_async") + async def test_raises_on_failure(self, mock_cmd): + mock_cmd.return_value = (1, "", "error occurred") + with self.assertRaises(RuntimeError): + await fetch_payload_images("quay.io/ocp:4.20.1") + + +class TestFetchImageIdentifiers(IsolatedAsyncioTestCase): + @patch("doozerlib.cli.verify_image_consistency.oc_image_info__cached_async") + async def test_parses_metadata(self, mock_oc): + oc_output = { + "digest": "sha256:img_digest", + "listDigest": "sha256:list_digest", + "config": { + "config": { + "Labels": { + "vcs-ref": "abc123", + "name": "ose-cli", + } + } + }, + } + mock_oc.return_value = json.dumps(oc_output) + + result = await fetch_image_identifiers("quay.io/ocp/cli@sha256:abc") + self.assertEqual(result.digest, "sha256:img_digest") + self.assertEqual(result.list_digest, "sha256:list_digest") + self.assertEqual(result.vcs_ref, "abc123") + self.assertEqual(result.name, "ose-cli") + + @patch("doozerlib.cli.verify_image_consistency.oc_image_info__cached_async") + async def test_returns_empty_on_error(self, mock_oc): + mock_oc.side_effect = Exception("oc failed") + + result = await fetch_image_identifiers("quay.io/ocp/cli@sha256:abc") + self.assertEqual(result.digest, "") + self.assertEqual(result.list_digest, "") + self.assertEqual(result.vcs_ref, "") + + +class TestVerifyImageConsistency(IsolatedAsyncioTestCase): + @patch("doozerlib.cli.verify_image_consistency.check_catalog") + @patch("doozerlib.cli.verify_image_consistency.fetch_image_identifiers") + @patch("doozerlib.cli.verify_image_consistency.fetch_shipment_components") + @patch("doozerlib.cli.verify_image_consistency.fetch_payload_images") + async def test_all_images_match(self, mock_payload, mock_shipment, mock_identifiers, mock_catalog): + mock_payload.return_value = ( + [ + ("ose-cli", "quay.io/ocp/cli@sha256:aaa"), + ("machine-os-content", "quay.io/ocp/rhcos@sha256:bbb"), + ], + "4.20.1", + ) + mock_shipment.return_value = ( + [("cli", "registry.redhat.io/ocp/cli@sha256:ccc")], + "4.20.1", + ) + + async def mock_id(pullspec): + return ImageIdentifiers(pullspec=pullspec, digest="sha256:shared_digest") + + mock_identifiers.side_effect = mock_id + mock_catalog.return_value = False + + result = await verify_image_consistency("quay.io/ocp:4.20.1", "https://gitlab.example.com/mr/1") + + self.assertTrue(result.passed) + self.assertEqual(len(result.skipped_images), 1) + self.assertIn("machine-os-content", result.skipped_images) + self.assertEqual(len(result.results), 1) + self.assertEqual(result.results[0].found_in, "shipment") + + @patch("doozerlib.cli.verify_image_consistency.check_catalog") + @patch("doozerlib.cli.verify_image_consistency.fetch_image_identifiers") + @patch("doozerlib.cli.verify_image_consistency.fetch_shipment_components") + @patch("doozerlib.cli.verify_image_consistency.fetch_payload_images") + async def test_image_not_found(self, mock_payload, mock_shipment, mock_identifiers, mock_catalog): + mock_payload.return_value = ( + [("ose-cli", "quay.io/ocp/cli@sha256:aaa")], + "4.20.1", + ) + mock_shipment.return_value = ( + [("other", "registry.redhat.io/ocp/other@sha256:bbb")], + "4.20.1", + ) + + async def mock_id(pullspec): + if "cli" in pullspec: + return ImageIdentifiers(pullspec=pullspec, digest="sha256:cli_digest") + return ImageIdentifiers(pullspec=pullspec, digest="sha256:other_digest") + + mock_identifiers.side_effect = mock_id + mock_catalog.return_value = False + + result = await verify_image_consistency("quay.io/ocp:4.20.1", "https://gitlab.example.com/mr/1") + + self.assertFalse(result.passed) + self.assertEqual(len(result.failed_images), 1) + self.assertEqual(result.failed_images[0].name, "ose-cli") + + @patch("doozerlib.cli.verify_image_consistency.check_catalog") + @patch("doozerlib.cli.verify_image_consistency.fetch_image_identifiers") + @patch("doozerlib.cli.verify_image_consistency.fetch_shipment_components") + @patch("doozerlib.cli.verify_image_consistency.fetch_payload_images") + async def test_catalog_fallback(self, mock_payload, mock_shipment, mock_identifiers, mock_catalog): + mock_payload.return_value = ( + [("ose-cli", "quay.io/ocp/cli@sha256:aaa")], + "4.20.1", + ) + mock_shipment.return_value = ([], "4.20.1") + + async def mock_id(pullspec): + return ImageIdentifiers(pullspec=pullspec, digest="sha256:cli_digest") + + mock_identifiers.side_effect = mock_id + mock_catalog.return_value = True + + result = await verify_image_consistency("quay.io/ocp:4.20.1", "https://gitlab.example.com/mr/1") + + self.assertTrue(result.passed) + self.assertEqual(result.results[0].found_in, "catalog") + + +class TestRenderResult(IsolatedAsyncioTestCase): + def _make_result(self, passed=True): + results = [ImageCheckResult(name="ose-cli", pullspec="quay.io/ocp/cli@sha256:abc", found_in="shipment")] + if not passed: + results.append(ImageCheckResult(name="ose-api", pullspec="quay.io/ocp/api@sha256:def")) + return VerifyImageConsistencyResult( + payload_url="quay.io/ocp:4.20.1", + shipment_mr_url="https://gitlab.example.com/mr/1", + payload_version="4.20.1", + shipment_version="4.20.1", + payload_image_count=3, + shipment_component_count=2, + skipped_images=["machine-os-content"], + results=results, + ) + + def test_text_output_pass(self): + result = self._make_result(passed=True) + output = render_result(result, "text") + self.assertIn("PASS", output) + self.assertIn("1/1 passed", output) + self.assertNotIn("NOT FOUND", output) + + def test_text_output_fail(self): + result = self._make_result(passed=False) + output = render_result(result, "text") + self.assertIn("FAIL", output) + self.assertIn("NOT FOUND", output) + self.assertIn("ose-api", output) + + def test_json_output(self): + result = self._make_result(passed=True) + output = render_result(result, "json") + data = json.loads(output) + self.assertTrue(data["passed"]) + self.assertEqual(data["payload_version"], "4.20.1") + self.assertEqual(len(data["results"]), 1) + self.assertEqual(data["results"][0]["name"], "ose-cli") + + def test_json_output_fail(self): + result = self._make_result(passed=False) + output = render_result(result, "json") + data = json.loads(output) + self.assertFalse(data["passed"]) + self.assertEqual(len(data["failed_images"]), 1) + self.assertEqual(data["failed_images"][0]["name"], "ose-api") From 28466100a42eae4e88288c884c08acac522d35ab Mon Sep 17 00:00:00 2001 From: Tomas David Date: Thu, 30 Jul 2026 15:44:57 +0200 Subject: [PATCH 2/5] ART-21809: Address PR review feedback - Guard against empty MR diff versions list with descriptive error - Fix Catalog API filter: use docker_image_digest instead of image_id - Use asyncio.gather for concurrent payload/shipment fetch and image identifier lookups instead of sequential awaits Co-Authored-By: Claude Opus 4.6 rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED --- .../doozerlib/cli/verify_image_consistency.py | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/doozer/doozerlib/cli/verify_image_consistency.py b/doozer/doozerlib/cli/verify_image_consistency.py index 2602e625f9..0e59440f9e 100644 --- a/doozer/doozerlib/cli/verify_image_consistency.py +++ b/doozer/doozerlib/cli/verify_image_consistency.py @@ -110,7 +110,10 @@ def fetch_shipment_components(mr_url: str) -> tuple[list[tuple[str, str]], str]: match = re.search(r"Shipment for (\d+\.\d+\.\d+(?:-\S+)?)", title, re.IGNORECASE) version = match.group(1) if match else "" - diff_info = mr.diffs.list(all=True)[0] + diff_versions = mr.diffs.list(all=True) + if not diff_versions: + raise RuntimeError(f"No diff versions found for MR {mr_url}") + diff_info = diff_versions[0] diff = mr.diffs.get(diff_info.id) components: list[tuple[str, str]] = [] @@ -161,7 +164,7 @@ async def check_catalog(digest: str) -> bool: if not digest: return False - url = f"{CATALOG_API_URL}?filter=image_id=={digest}" + url = f"{CATALOG_API_URL}?filter=docker_image_digest=={digest}" try: async with aiohttp.ClientSession() as session: async with session.get(url) as resp: @@ -176,11 +179,10 @@ async def check_catalog(digest: str) -> bool: async def verify_image_consistency(payload_url: str, shipment_mr_url: str) -> VerifyImageConsistencyResult: - payload_images_task = fetch_payload_images(payload_url) - shipment_task = asyncio.to_thread(fetch_shipment_components, shipment_mr_url) - - payload_images, payload_version = await payload_images_task - shipment_components, shipment_version = await shipment_task + (payload_images, payload_version), (shipment_components, shipment_version) = await asyncio.gather( + fetch_payload_images(payload_url), + asyncio.to_thread(fetch_shipment_components, shipment_mr_url), + ) result = VerifyImageConsistencyResult( payload_url=payload_url, @@ -212,10 +214,9 @@ async def verify_image_consistency(payload_url: str, shipment_mr_url: str) -> Ve for _, pullspec in shipment_components: all_pullspecs.add(pullspec) - identifiers_tasks = {ps: fetch_image_identifiers(ps) for ps in all_pullspecs} - identifiers: dict[str, ImageIdentifiers] = {} - for ps, task in identifiers_tasks.items(): - identifiers[ps] = await task + pullspec_list = list(all_pullspecs) + fetched = await asyncio.gather(*(fetch_image_identifiers(ps) for ps in pullspec_list)) + identifiers: dict[str, ImageIdentifiers] = dict(zip(pullspec_list, fetched)) shipment_identifiers = [identifiers[ps] for _, ps in shipment_components if ps in identifiers] From bb55654f0ef209078ff908627004d16d8159c219 Mon Sep 17 00:00:00 2001 From: Tomas David Date: Thu, 30 Jul 2026 16:00:28 +0200 Subject: [PATCH 3/5] ART-21809: Address nitpick review comments - Add explicit 15s timeout for Red Hat Catalog API requests - Use strict=True in zip() for pullspec/identifiers mapping Co-Authored-By: Claude Opus 4.6 rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED --- doozer/doozerlib/cli/verify_image_consistency.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doozer/doozerlib/cli/verify_image_consistency.py b/doozer/doozerlib/cli/verify_image_consistency.py index 0e59440f9e..2728e1bc08 100644 --- a/doozer/doozerlib/cli/verify_image_consistency.py +++ b/doozer/doozerlib/cli/verify_image_consistency.py @@ -166,7 +166,8 @@ async def check_catalog(digest: str) -> bool: url = f"{CATALOG_API_URL}?filter=docker_image_digest=={digest}" try: - async with aiohttp.ClientSession() as session: + timeout = aiohttp.ClientTimeout(total=15) + async with aiohttp.ClientSession(timeout=timeout) as session: async with session.get(url) as resp: if resp.status != 200: LOGGER.warning("Red Hat Catalog API returned status %s", resp.status) @@ -216,7 +217,7 @@ async def verify_image_consistency(payload_url: str, shipment_mr_url: str) -> Ve pullspec_list = list(all_pullspecs) fetched = await asyncio.gather(*(fetch_image_identifiers(ps) for ps in pullspec_list)) - identifiers: dict[str, ImageIdentifiers] = dict(zip(pullspec_list, fetched)) + identifiers: dict[str, ImageIdentifiers] = dict(zip(pullspec_list, fetched, strict=True)) shipment_identifiers = [identifiers[ps] for _, ps in shipment_components if ps in identifiers] From 9dac3e78ea14df98849246dafb610750d86adab0 Mon Sep 17 00:00:00 2001 From: Tomas David Date: Fri, 31 Jul 2026 09:58:17 +0200 Subject: [PATCH 4/5] ART-21809: Throttle concurrent oc image info calls with semaphore Co-Authored-By: Claude Opus 4.6 rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED --- doozer/doozerlib/cli/verify_image_consistency.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/doozer/doozerlib/cli/verify_image_consistency.py b/doozer/doozerlib/cli/verify_image_consistency.py index 2728e1bc08..c262301834 100644 --- a/doozer/doozerlib/cli/verify_image_consistency.py +++ b/doozer/doozerlib/cli/verify_image_consistency.py @@ -216,7 +216,13 @@ async def verify_image_consistency(payload_url: str, shipment_mr_url: str) -> Ve all_pullspecs.add(pullspec) pullspec_list = list(all_pullspecs) - fetched = await asyncio.gather(*(fetch_image_identifiers(ps) for ps in pullspec_list)) + semaphore = asyncio.Semaphore(50) + + async def _throttled_fetch(ps: str) -> ImageIdentifiers: + async with semaphore: + return await fetch_image_identifiers(ps) + + fetched = await asyncio.gather(*(_throttled_fetch(ps) for ps in pullspec_list)) identifiers: dict[str, ImageIdentifiers] = dict(zip(pullspec_list, fetched, strict=True)) shipment_identifiers = [identifiers[ps] for _, ps in shipment_components if ps in identifiers] From a5b5a5ceb3ac231ad30a9321567118a8bcc8311b Mon Sep 17 00:00:00 2001 From: Tomas David Date: Fri, 31 Jul 2026 16:07:18 +0200 Subject: [PATCH 5/5] ART-21809: Use global --group/--assembly instead of explicit URLs for verify-image-consistency Resolve shipment MR URL from assembly config and construct payload pullspec from assembly name, consistent with verify-image-grades pattern. Co-Authored-By: Claude Opus 4.6 rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED --- .../doozerlib/cli/verify_image_consistency.py | 39 ++++++++++++++----- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/doozer/doozerlib/cli/verify_image_consistency.py b/doozer/doozerlib/cli/verify_image_consistency.py index c262301834..43facf660c 100644 --- a/doozer/doozerlib/cli/verify_image_consistency.py +++ b/doozer/doozerlib/cli/verify_image_consistency.py @@ -9,6 +9,7 @@ import click import yaml from artcommonlib import exectools +from artcommonlib.assembly import assembly_config_struct from artcommonlib.gitlab import GitLabClient from artcommonlib.oc_image_info import oc_image_info__cached_async @@ -23,6 +24,7 @@ ] CATALOG_API_URL = "https://catalog.redhat.com/api/containers/v1/images" +RELEASE_IMAGE_REPO = "quay.io/openshift-release-dev/ocp-release" @dataclass @@ -294,13 +296,26 @@ def render_result(result: VerifyImageConsistencyResult, output: str) -> str: return "\n".join(lines) +def resolve_shipment_mr_url(runtime) -> str: + releases_config = runtime.get_releases_config() + assembly_group_config = assembly_config_struct(releases_config, runtime.assembly, "group", {}) + shipment = assembly_group_config.get("shipment", {}) + url = shipment.get("url") + if not url: + raise RuntimeError( + f"No shipment URL found in assembly '{runtime.assembly}' group config. " + f"Ensure releases.yml has releases.{runtime.assembly}.assembly.group.shipment.url set." + ) + return url + + @cli.command("verify-image-consistency", short_help="Verify payload images match shipment MR components") @click.option( - "--payload-url", - required=True, - help="Release payload pullspec, e.g. quay.io/openshift-release-dev/ocp-release:4.20.1-x86_64", + "--arch", + default="x86_64", + show_default=True, + help="Architecture for the release payload.", ) -@click.option("--shipment-mr-url", required=True, help="Shipment GitLab MR URL") @click.option( "-o", "--output", @@ -311,15 +326,21 @@ def render_result(result: VerifyImageConsistencyResult, output: str) -> str: ) @pass_runtime @click_coroutine -async def verify_image_consistency_cli(runtime, payload_url, shipment_mr_url, output): +async def verify_image_consistency_cli(runtime, arch, output): """Verify that every image in the release payload is present in the shipment MR or has already been released in the Red Hat catalog. - Compares payload images (from oc adm release info) against shipment - components (from the GitLab MR YAML files). RHCOS images are skipped. - Images are matched by digest, list digest, or VCS reference. + Requires --group and --assembly global options. Resolves the shipment + MR URL from the assembly config and constructs the payload pullspec. + + Example: + doozer --group openshift-4.18 --assembly 4.18.51 verify-image-consistency """ - runtime.initialize(no_group=True) + runtime.initialize(config_only=True) + shipment_mr_url = resolve_shipment_mr_url(runtime) + payload_url = f"{RELEASE_IMAGE_REPO}:{runtime.assembly}-{arch}" + LOGGER.info("Resolved shipment MR URL: %s", shipment_mr_url) + LOGGER.info("Constructed payload URL: %s", payload_url) result = await verify_image_consistency(payload_url=payload_url, shipment_mr_url=shipment_mr_url) click.echo(render_result(result, output)) if not result.passed: