From 3c38d21ef350ac100ed856b4d50f6f0e2f9f5504 Mon Sep 17 00:00:00 2001 From: LuLo Date: Mon, 29 Jun 2026 11:28:10 +0200 Subject: [PATCH 1/2] feat(pyartcd): add golang-builder-shipment pipeline New artcd command to create shipment MRs in ocp-shipment-data for golang builder images. Auto-resolves Konflux image NVRs from golang RPM NVRs, detects prod vs ec lifecycle from ocp-build-data, and opens a draft MR for ERT approval. rh-pre-commit.version: 2.3.2 rh-pre-commit.check-secrets: ENABLED --- doozer/doozerlib/assembly_inspector.py | 38 +- doozer/doozerlib/backend/konflux_fbc.py | 3 +- doozer/doozerlib/backend/rebaser.py | 3 +- doozer/doozerlib/cli/scan_sources_konflux.py | 58 +- .../doozerlib/lockfile_prototype/constants.py | 3 +- .../lockfile_prototype/rebaser_hooks.py | 2 - .../doozerlib/lockfile_prototype/resolver.py | 23 +- doozer/tests/backend/test_konflux_client.py | 5 +- doozer/tests/backend/test_konflux_fbc.py | 13 +- doozer/tests/cli/test_images_streams.py | 66 +- .../tests/lockfile_prototype/test_resolver.py | 98 --- .../lockfile_prototype/test_shell_parser.py | 64 +- doozer/tests/test_assembly_inspector.py | 23 - .../cli/process_release_from_fbc_bugs_cli.py | 76 +-- .../test_process_release_from_fbc_bugs_cli.py | 199 +----- .../tests/test_schema/test_group_schema.py | 23 - .../validator/json_schemas/repos.schema.json | 1 - pyartcd/pyartcd/__main__.py | 2 + pyartcd/pyartcd/pipelines/__init__.py | 2 + .../pipelines/build_microshift_bootc.py | 7 +- .../pipelines/golang_builder_shipment.py | 519 +++++++++++++++ pyartcd/pyartcd/pipelines/okd.py | 2 +- .../pyartcd/pipelines/quay_doomsday_backup.py | 10 +- pyartcd/pyartcd/pipelines/sync_ci_images.py | 49 +- .../pipelines/test_build_microshift_bootc.py | 53 -- .../pipelines/test_golang_builder_shipment.py | 627 ++++++++++++++++++ pyartcd/tests/pipelines/test_images_health.py | 74 +-- .../pipelines/test_quay_doomsday_backup.py | 59 -- .../tests/pipelines/test_sync_ci_images.py | 60 -- 29 files changed, 1260 insertions(+), 902 deletions(-) create mode 100644 pyartcd/pyartcd/pipelines/golang_builder_shipment.py create mode 100644 pyartcd/tests/pipelines/test_golang_builder_shipment.py delete mode 100644 pyartcd/tests/pipelines/test_quay_doomsday_backup.py diff --git a/doozer/doozerlib/assembly_inspector.py b/doozer/doozerlib/assembly_inspector.py index 681a40554c..2a154b9748 100644 --- a/doozer/doozerlib/assembly_inspector.py +++ b/doozer/doozerlib/assembly_inspector.py @@ -202,20 +202,12 @@ async def check_rhcos_issues(self, rhcos_build: RHCOSBuildInspector) -> List[Ass build_dict = external_rpms.get(entry["tag"], {}).get(package_name) if not build_dict: continue - installed_rpm = installed_packages.get(package_name) - if installed_rpm and installed_rpm["nvr"] != build_dict["nvr"]: - if not self._is_installed_rpm_in_tag(installed_rpm, entry["tag"]): - self.runtime.logger.info( - "Installed rpm %s in RHCOS %s (%s) is not tagged in %s; skipping check_external_packages complaint", - installed_rpm["nvr"], - rhcos_build.build_id, - rhcos_build.brew_arch, - entry["tag"], - ) - continue if entry["condition"] == "match": desired_packages[package_name] = build_dict["nvr"] elif entry["condition"] == "greater_equal": + # installed_rpm = installed_packages.get(package_name) + # desired_packages[package_name] = build_dict["nvr"] if not installed_rpm or compare_nvr(build_dict, installed_rpm) > 0 else installed_rpm["nvr"] + installed_rpm = installed_packages.get(package_name) if not installed_rpm or compare_nvr(build_dict, installed_rpm) >= 0: desired_packages[package_name] = build_dict["nvr"] else: @@ -430,19 +422,10 @@ def check_group_image_consistency( build_dict = external_rpms.get(entry["tag"], {}).get(package_name) if not build_dict: continue - installed_rpm = installed_packages.get(package_name) - if installed_rpm and installed_rpm["nvr"] != build_dict["nvr"]: - if not self._is_installed_rpm_in_tag(installed_rpm, entry["tag"]): - image_meta.logger.info( - "Installed rpm %s in image %s is not tagged in %s; skipping check_external_packages complaint", - installed_rpm["nvr"], - dgk, - entry["tag"], - ) - continue if entry["condition"] == "match": desired_packages[package_name] = build_dict["nvr"] elif entry["condition"] == "greater_equal": + installed_rpm = installed_packages.get(package_name) if not installed_rpm or compare_nvr(build_dict, installed_rpm) >= 0: desired_packages[package_name] = build_dict["nvr"] else: @@ -586,19 +569,6 @@ def get_group_rpm_build_dicts(self, el_ver: int) -> Dict[str, Optional[Dict]]: return self._rpm_build_cache[el_ver] - def _is_installed_rpm_in_tag(self, installed_rpm: Optional[Dict], tag: str) -> bool: - """Check if an installed RPM build is tagged in the given Brew tag. - :param installed_rpm: installed package build dict (must have 'id' key), or None - :param tag: Brew tag name to check - :return: True if the installed RPM is tagged in the given tag - """ - if not installed_rpm: - return False - tag_names = { - t["name"] for t in self.brew_session.listTags(brew.KojiWrapperOpts(caching=True), build=installed_rpm["id"]) - } - return tag in tag_names - def get_external_rpm_build_dicts(self): """Get external rpm build dicts from rhaos candidate Brew tags. :return: a dict. key is Brew tag name, value is another (component_name, rpm_build) dict diff --git a/doozer/doozerlib/backend/konflux_fbc.py b/doozer/doozerlib/backend/konflux_fbc.py index 07267ebacc..6804a8e7d4 100644 --- a/doozer/doozerlib/backend/konflux_fbc.py +++ b/doozer/doozerlib/backend/konflux_fbc.py @@ -1602,8 +1602,7 @@ def _catagorize_catalog_blobs(self, blobs: List[Dict]): raise IOError(f"Found unsupported schema: {schema}") if not package_name: raise IOError(f"Couldn't determine package name for unknown schema: {schema}") - blob_key = package_name if schema == "olm.deprecations" else blob["name"] - categorized_blobs.setdefault(package_name, {}).setdefault(schema, {})[blob_key] = blob + categorized_blobs.setdefault(package_name, {}).setdefault(schema, {})[blob["name"]] = blob return categorized_blobs diff --git a/doozer/doozerlib/backend/rebaser.py b/doozer/doozerlib/backend/rebaser.py index 49d3e28ced..99a306662c 100644 --- a/doozer/doozerlib/backend/rebaser.py +++ b/doozer/doozerlib/backend/rebaser.py @@ -1042,10 +1042,9 @@ async def _update_build_dir( from doozerlib.lockfile_prototype.rebaser_hooks import apply_dockerfile_transforms lockfile_has_packages = bool(getattr(metadata, "lockfile_packages", None)) - upgrades_dropped = getattr(metadata, "lockfile_upgrades_dropped", False) apply_dockerfile_transforms( dest_dir, - strip_updates=not downstream_parents or not lockfile_has_packages or upgrades_dropped, + strip_updates=not downstream_parents or not lockfile_has_packages, logger=self._logger, ) diff --git a/doozer/doozerlib/cli/scan_sources_konflux.py b/doozer/doozerlib/cli/scan_sources_konflux.py index 7f06dba72d..7cb37441b2 100644 --- a/doozer/doozerlib/cli/scan_sources_konflux.py +++ b/doozer/doozerlib/cli/scan_sources_konflux.py @@ -24,7 +24,7 @@ from artcommonlib.pushd import Dir from artcommonlib.release_util import isolate_timestamp_in_release from artcommonlib.rhcos import get_latest_layered_rhcos_build, get_primary_container_name -from artcommonlib.rpm_utils import parse_nvr, to_nevra +from artcommonlib.rpm_utils import parse_nvr from artcommonlib.util import deep_merge, fetch_slsa_attestation, uses_konflux_imagestream_override from artcommonlib.variants import BuildVariant from async_lru import alru_cache @@ -629,12 +629,11 @@ async def scan_image(self, image_meta: ImageMetadata): stage = 'external image checks' await self.scan_external_image_changes(image_meta) - # Check for changes in image arches - stage = 'arch checks' - await self.scan_arch_changes(image_meta) - # For OCP variant, perform additional checks that don't apply to OKD if self.variant != BuildVariant.OKD: + # Check for changes in image arches (skip for OKD - arch changes don't trigger OKD rebuilds) + stage = 'arch checks' + await self.scan_arch_changes(image_meta) # Check for changes in the network mode (skip for OKD - it always uses open network) stage = 'network mode checks' await self.scan_network_mode_changes(image_meta) @@ -1277,10 +1276,6 @@ async def scan_rpm_changes(self, image_meta: ImageMetadata): # Check for changes in non-ART RPMs build_record_inspector = KonfluxBuildRecordInspector(self.runtime, build_record) non_latest_rpms = await build_record_inspector.find_non_latest_rpms(self.package_rpm_finder) - - if non_latest_rpms: - non_latest_rpms = await self._filter_parent_inherited_rpms(image_meta, non_latest_rpms) - rebuild_hints = [ f"Outdated RPM {installed_rpm} installed in {build_record.nvr} ({arch}) when {latest_rpm} was available in repo {repo}" for arch, non_latest in non_latest_rpms.items() @@ -1293,51 +1288,6 @@ async def scan_rpm_changes(self, image_meta: ImageMetadata): else: self.logger.info('No package changes detected for %s', build_record.nvr) - async def _filter_parent_inherited_rpms( - self, - image_meta: ImageMetadata, - non_latest_rpms: dict[str, list[tuple[str, str, str]]], - ) -> dict[str, list[tuple[str, str, str]]]: - """ - Filter out outdated RPMs inherited from the parent image. - - If an outdated RPM is at the same version as in the parent image, - rebuilding this image cannot fix it — the parent must be updated - first. Suppressing these prevents pointless rebuild loops. - """ - parent_key = image_meta.config["from"].member - if not parent_key: - return non_latest_rpms - - parent_build = self.latest_image_build_records_map.get(parent_key) - if not parent_build: - return non_latest_rpms - - parent_rpms = self.package_rpm_finder.get_brew_rpms_from_build_record(parent_build) - parent_nevras = {to_nevra(rpm) for rpm in parent_rpms} - - filtered = {} - suppressed = 0 - for arch, rpm_list in non_latest_rpms.items(): - kept = [] - for installed_rpm, latest_rpm, repo in rpm_list: - if installed_rpm in parent_nevras: - suppressed += 1 - else: - kept.append((installed_rpm, latest_rpm, repo)) - if kept: - filtered[arch] = kept - - if suppressed: - self.logger.info( - "%s: suppressed %d outdated RPMs inherited from parent %s", - image_meta.distgit_key, - suppressed, - parent_key, - ) - - return filtered - @skip_check_if_changing async def scan_extra_packages(self, image_meta: ImageMetadata): """ diff --git a/doozer/doozerlib/lockfile_prototype/constants.py b/doozer/doozerlib/lockfile_prototype/constants.py index 2ff2010874..0e54785ec5 100644 --- a/doozer/doozerlib/lockfile_prototype/constants.py +++ b/doozer/doozerlib/lockfile_prototype/constants.py @@ -35,7 +35,7 @@ # RPM pseudo-packages that appear in rpmdb but are not installable via DNF RPM_PSEUDO_PACKAGES = frozenset({"gpg-pubkey"}) -VALID_PKG_NAME = re.compile(r"^[a-zA-Z0-9*][a-zA-Z0-9._+\-*]*$") +VALID_PKG_NAME = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9._+\-]*$") # rpm-lockfile-prototype stores extracted RPMDBs under @@ -53,7 +53,6 @@ RPMDB_CACHE_ERROR_PATTERNS = [ "database disk image is malformed", "failed loading RPMDB", - "No such file or directory", ] diff --git a/doozer/doozerlib/lockfile_prototype/rebaser_hooks.py b/doozer/doozerlib/lockfile_prototype/rebaser_hooks.py index 4bbcced965..59f7b573c7 100644 --- a/doozer/doozerlib/lockfile_prototype/rebaser_hooks.py +++ b/doozer/doozerlib/lockfile_prototype/rebaser_hooks.py @@ -93,8 +93,6 @@ async def generate_lockfile( pkg_names.add(name) metadata.lockfile_packages = sorted(pkg_names) - metadata.lockfile_upgrades_dropped = generator.upgrades_dropped - return shared_dnf_cache diff --git a/doozer/doozerlib/lockfile_prototype/resolver.py b/doozer/doozerlib/lockfile_prototype/resolver.py index 9060f8609c..7a79ab126e 100644 --- a/doozer/doozerlib/lockfile_prototype/resolver.py +++ b/doozer/doozerlib/lockfile_prototype/resolver.py @@ -103,14 +103,13 @@ async def resolve( if rc != 0: if image_pullspec and self._is_rpmdb_corrupt(stderr): - self._clear_rpmdb_cache(image_pullspec) - self.logger.info("Retrying rpm-lockfile-prototype after RPMDB cache error") - rc, _, stderr = await cmd_gather_async(cmd, check=False, env=env) - if rc == 0: - return LockfileData.model_validate(yaml.safe_load(out_file.read_text())) - error_summary = stderr.strip().rsplit("\n", 1)[-1] - self.logger.warning("Retry also failed (exit code %d): %s", rc, error_summary) - self.logger.debug("Full retry stderr:\n%s", stderr) + cleared = self._clear_rpmdb_cache(image_pullspec) + if cleared: + self.logger.info("Retrying rpm-lockfile-prototype after clearing corrupt RPMDB cache") + retry_rc, _, retry_stderr = await cmd_gather_async(cmd, check=False, env=env) + if retry_rc == 0: + return LockfileData.model_validate(yaml.safe_load(out_file.read_text())) + self.logger.warning("Retry also failed (exit code %d): %s", retry_rc, retry_stderr) raise RuntimeError(f"rpm-lockfile-prototype failed (exit code {rc}): {stderr}") @@ -168,9 +167,8 @@ def parse_missing_packages(error_text: str) -> set[str]: """ Parse missing package names from rpm-lockfile-prototype error output. - Handles the CLI format ("missing packages: X, Y"), DNF install/upgrade - errors ("No match for argument: X"), and DNF reinstall errors - ("no package matched: X"). + Handles both the CLI format ("missing packages: X, Y") and the + DNF format ("No match for argument: X"). Arg(s): error_text (str): Error message from rpm-lockfile-prototype. @@ -185,7 +183,4 @@ def parse_missing_packages(error_text: str) -> set[str]: m = re.search(r"No match for argument:\s*(\S+)", line.strip()) if m: missing.add(m.group(1).strip().rstrip(":")) - m = re.search(r"no package matched:\s*(\S+)", line.strip()) - if m: - missing.add(m.group(1).strip().rstrip(":")) return {p for p in missing if VALID_PKG_NAME.match(p)} diff --git a/doozer/tests/backend/test_konflux_client.py b/doozer/tests/backend/test_konflux_client.py index 58207b7db1..3a6b061245 100644 --- a/doozer/tests/backend/test_konflux_client.py +++ b/doozer/tests/backend/test_konflux_client.py @@ -1015,10 +1015,7 @@ class TestSkipTasks(IsolatedAsyncioTestCase): "taskRunTemplate": {"serviceAccountName": "default"}, "pipelineSpec": { "tasks": [ - { - "name": "clone-repository", - "params": [{"name": "refspec", "value": ""}, {"name": "fetchTags", "value": "false"}], - }, + {"name": "clone-repository", "params": [{"name": "refspec", "value": ""}]}, {"name": "build-images", "params": [{"name": "SBOM_TYPE", "value": ""}]}, {"name": "clair-scan", "params": []}, {"name": "sast-snyk-check", "params": []}, diff --git a/doozer/tests/backend/test_konflux_fbc.py b/doozer/tests/backend/test_konflux_fbc.py index a78c61c977..e127038c5c 100644 --- a/doozer/tests/backend/test_konflux_fbc.py +++ b/doozer/tests/backend/test_konflux_fbc.py @@ -680,27 +680,17 @@ async def test_fetch_olm_bundle_blob(self, mock_render): mock_render.assert_called_once_with("test-image-pullspec", migrate_level="none", auth=ANY) def test_categorize_catalog_blobs(self): - deprecation_blob = { - "schema": "olm.deprecations", - "package": "test-package", - "entries": [ - {"reference": {"schema": "olm.bundle", "name": "test-bundle.v1.0.0"}, "message": "deprecated"}, - ], - } catalog_blobs = [ {"schema": "olm.package", "name": "test-package"}, {"schema": "olm.channel", "name": "test-channel", "package": "test-package"}, {"schema": "olm.bundle", "name": "test-bundle", "package": "test-package"}, - deprecation_blob, {"schema": "olm.package", "name": "test-package2"}, {"schema": "olm.channel", "name": "test-channel2", "package": "test-package2"}, {"schema": "olm.bundle", "name": "test-bundle2", "package": "test-package2"}, ] actual = self.rebaser._catagorize_catalog_blobs(catalog_blobs) self.assertEqual(actual.keys(), {"test-package", "test-package2"}) - self.assertEqual( - actual["test-package"].keys(), {"olm.package", "olm.channel", "olm.bundle", "olm.deprecations"} - ) + self.assertEqual(actual["test-package"].keys(), {"olm.package", "olm.channel", "olm.bundle"}) self.assertEqual( actual["test-package"]["olm.package"]["test-package"], {"schema": "olm.package", "name": "test-package"} ) @@ -712,7 +702,6 @@ def test_categorize_catalog_blobs(self): actual["test-package"]["olm.bundle"]["test-bundle"], {"schema": "olm.bundle", "name": "test-bundle", "package": "test-package"}, ) - self.assertEqual(actual["test-package"]["olm.deprecations"]["test-package"], deprecation_blob) self.assertEqual(actual["test-package2"].keys(), {"olm.package", "olm.channel", "olm.bundle"}) self.assertEqual( actual["test-package2"]["olm.package"]["test-package2"], {"schema": "olm.package", "name": "test-package2"} diff --git a/doozer/tests/cli/test_images_streams.py b/doozer/tests/cli/test_images_streams.py index 378cc5c7f9..079e774318 100644 --- a/doozer/tests/cli/test_images_streams.py +++ b/doozer/tests/cli/test_images_streams.py @@ -278,70 +278,6 @@ def test_get_upstreaming_entries_with_final_user(mock_runtime, mock_image_meta): assert result['ose-test'].final_user == '1001' -# Tests for --image filtering in _get_upstreaming_entries - - -def test_get_upstreaming_entries_image_names_only(mock_runtime, mock_image_meta): - """Test filtering by image_names skips streams and only processes specified images.""" - mock_image_meta.distgit_key = 'ci-openshift-base.rhel10' - mock_image_meta.pull_url.return_value = 'brew-registry.example.com/ci-openshift-base:latest' - mock_runtime.image_map = {'ci-openshift-base.rhel10': mock_image_meta} - - result = images_streams._get_upstreaming_entries(mock_runtime, image_names=['ci-openshift-base.rhel10']) - - assert len(result) == 1 - assert 'ci-openshift-base.rhel10' in result - mock_runtime.ordered_image_metas.assert_not_called() - mock_runtime.get_stream_names.assert_not_called() - - -def test_get_upstreaming_entries_image_not_found(mock_runtime): - """Test error when specified image_name is not in the group.""" - mock_runtime.image_map = {} - - with pytest.raises(IOError, match='Image nonexistent not found in group metadata'): - images_streams._get_upstreaming_entries(mock_runtime, image_names=['nonexistent']) - - -def test_get_upstreaming_entries_image_not_eligible(mocker, mock_runtime): - """Test error when image lacks ci_alignment.upstream_image.""" - ineligible_meta = mocker.MagicMock() - ineligible_meta.config.content.source.ci_alignment.upstream_image = Missing - mock_runtime.image_map = {'ose-cli': ineligible_meta} - - with pytest.raises(IOError, match='not eligible for CI sync'): - images_streams._get_upstreaming_entries(mock_runtime, image_names=['ose-cli']) - - -def test_get_upstreaming_entries_stream_and_image_union(mocker, mock_runtime, mock_image_meta): - """Test that providing both stream_names and image_names returns the union.""" - mock_runtime.streams = { - 'golang': Model({'upstream_image': 'registry.ci.openshift.org/ocp/4.17:golang'}), - } - mock_image_meta.distgit_key = 'ci-openshift-base.rhel10' - mock_image_meta.pull_url.return_value = 'brew-registry.example.com/ci-openshift-base:latest' - mock_runtime.image_map = {'ci-openshift-base.rhel10': mock_image_meta} - - result = images_streams._get_upstreaming_entries( - mock_runtime, stream_names=['golang'], image_names=['ci-openshift-base.rhel10'] - ) - - assert len(result) == 2 - assert 'golang' in result - assert 'ci-openshift-base.rhel10' in result - - -def test_get_upstreaming_entries_image_build_not_found_skips(mocker, mock_runtime, mock_image_meta): - """Test that an image with no build is gracefully skipped.""" - mock_image_meta.distgit_key = 'ci-openshift-base.rhel10' - mock_image_meta.pull_url.side_effect = IOError('No build found') - mock_runtime.image_map = {'ci-openshift-base.rhel10': mock_image_meta} - - result = images_streams._get_upstreaming_entries(mock_runtime, image_names=['ci-openshift-base.rhel10']) - - assert result == {} - - # Tests for images:streams gen-buildconfigs command @@ -359,7 +295,7 @@ def test_gen_buildconfigs_uses_get_upstreaming_entries(): source = inspect.getsource(callback) # Verify it calls _get_upstreaming_entries - assert '_get_upstreaming_entries(runtime, streams' in source + assert '_get_upstreaming_entries(runtime, streams)' in source # Verify it uses the configuration fields from entries assert 'config.transform' in source diff --git a/doozer/tests/lockfile_prototype/test_resolver.py b/doozer/tests/lockfile_prototype/test_resolver.py index 0faddc0974..bc12ada95a 100644 --- a/doozer/tests/lockfile_prototype/test_resolver.py +++ b/doozer/tests/lockfile_prototype/test_resolver.py @@ -260,26 +260,6 @@ def test_packages_not_installed_error_format(self): missing = RpmResolver.parse_missing_packages(error) self.assertEqual(missing, {"policycoreutils-python-utils"}) - def test_reinstall_not_available_format(self): - """ - DNF PackagesNotAvailableError from base.reinstall() outputs - "no package matched: " when the installed version is not - in the configured repos. - """ - error = "dnf.exceptions.PackagesNotAvailableError: no package matched: git" - missing = RpmResolver.parse_missing_packages(error) - self.assertEqual(missing, {"git"}) - - def test_glob_pattern_in_missing_packages(self): - """ - DNF glob patterns like *-server-ose* can appear in missing packages - errors. VALID_PKG_NAME must accept wildcards so the retry loop can - strip them. - """ - error = "missing packages: *-server-ose*" - missing = RpmResolver.parse_missing_packages(error) - self.assertEqual(missing, {"*-server-ose*"}) - def test_no_match(self): error = "Some other error message\n" missing = RpmResolver.parse_missing_packages(error) @@ -299,15 +279,6 @@ def test_detects_failed_loading_rpmdb(self): stderr = "OSError: failed loading RPMDB\n" self.assertTrue(RpmResolver._is_rpmdb_corrupt(stderr)) - def test_detects_no_such_file(self): - stderr = ( - "shutil.Error: [('/home/jenkins/.cache/rpm-lockfile-prototype/rpmdbs/ppc64le/" - "sha256:abc123/var/lib/rpm/rpmdb.sqlite', '/tmp/xyz/var/lib/rpm/rpmdb.sqlite', " - "\"[Errno 2] No such file or directory: '/home/jenkins/.cache/rpm-lockfile-prototype/" - "rpmdbs/ppc64le/sha256:abc123/var/lib/rpm/rpmdb.sqlite'\")]" - ) - self.assertTrue(RpmResolver._is_rpmdb_corrupt(stderr)) - def test_no_false_positive(self): stderr = "No match for argument: foo\n" self.assertFalse(RpmResolver._is_rpmdb_corrupt(stderr)) @@ -425,44 +396,6 @@ async def mock_cmd(cmd, **kwargs): self.assertEqual(call_count, 2) mock_clear.assert_called_once() - @patch("doozerlib.lockfile_prototype.resolver.RpmResolver._clear_rpmdb_cache") - @patch("doozerlib.lockfile_prototype.resolver.cmd_gather_async") - def test_retries_when_cache_already_gone(self, mock_gather, mock_clear): - """ - Cache deleted by another process (_clear returns False) — should still retry. - """ - cache_race_stderr = ( - "shutil.Error: [('/home/jenkins/.cache/rpm-lockfile-prototype/rpmdbs/ppc64le/" - "sha256:abc123/var/lib/rpm/rpmdb.sqlite', '/tmp/xyz/var/lib/rpm/rpmdb.sqlite', " - "\"[Errno 2] No such file or directory: '/home/jenkins/.cache/rpm-lockfile-prototype/" - "rpmdbs/ppc64le/sha256:abc123/var/lib/rpm/rpmdb.sqlite'\")]" - ) - call_count = 0 - - async def mock_cmd(cmd, **kwargs): - nonlocal call_count - call_count += 1 - if call_count == 1: - return (1, "", cache_race_stderr) - outfile_idx = cmd.index("--outfile") + 1 - with open(cmd[outfile_idx], "w") as f: - yaml.safe_dump(self.FAKE_LOCKFILE_DATA, f) - return (0, "", "") - - mock_gather.side_effect = mock_cmd - mock_clear.return_value = False - - resolver = RpmResolver(working_dir=Path(tempfile.mkdtemp())) - config = RpmsInConfig( - arches=["x86_64"], - contentOrigin={"repos": []}, - packages=[], - ) - result = asyncio.run(resolver.resolve(config, image_pullspec="registry.example.com/repo@sha256:abc123")) - self.assertIsInstance(result, LockfileData) - self.assertEqual(call_count, 2) - mock_clear.assert_called_once() - @patch("doozerlib.lockfile_prototype.resolver.RpmResolver._clear_rpmdb_cache") @patch("doozerlib.lockfile_prototype.resolver.cmd_gather_async") def test_raises_after_retry_fails(self, mock_gather, mock_clear): @@ -529,34 +462,3 @@ async def mock_fail(cmd, **kwargs): with self.assertRaises(RuntimeError): asyncio.run(resolver.resolve(config, image_pullspec="registry.example.com/repo@sha256:abc123")) mock_clear.assert_not_called() - - @patch("doozerlib.lockfile_prototype.resolver.RpmResolver._clear_rpmdb_cache") - @patch("doozerlib.lockfile_prototype.resolver.cmd_gather_async") - def test_retry_raises_retry_error_not_original(self, mock_gather, mock_clear): - """ - When cache corruption triggers a retry and the retry fails with a - different error, the raised RuntimeError must contain the retry - stderr so the outer retry loop can parse it. - """ - retry_stderr = "dnf.exceptions.PackagesNotInstalledError: No match for argument: nfs-utils: nfs-utils" - call_count = 0 - - async def mock_cmd(cmd, **kwargs): - nonlocal call_count - call_count += 1 - if call_count == 1: - return (1, "", self.CORRUPTION_STDERR) - return (1, "", retry_stderr) - - mock_gather.side_effect = mock_cmd - mock_clear.return_value = True - - resolver = RpmResolver(working_dir=Path(tempfile.mkdtemp())) - config = RpmsInConfig( - arches=["x86_64"], - contentOrigin={"repos": []}, - packages=[], - ) - with self.assertRaises(RuntimeError) as ctx: - asyncio.run(resolver.resolve(config, image_pullspec="registry.example.com/repo@sha256:abc123")) - self.assertIn("nfs-utils", str(ctx.exception)) diff --git a/doozer/tests/lockfile_prototype/test_shell_parser.py b/doozer/tests/lockfile_prototype/test_shell_parser.py index d7a672fe80..190b5a2b4f 100644 --- a/doozer/tests/lockfile_prototype/test_shell_parser.py +++ b/doozer/tests/lockfile_prototype/test_shell_parser.py @@ -153,41 +153,12 @@ def test_subshell_arch_conditional_var(self): "yum -y install pciutils hwdata kmod $ARCH_DEP_PKGS" ] common, arch = extract_packages_from_run_commands(run_values) - self.assertNotIn("mstflint", common) + self.assertIn("mstflint", common) self.assertIn("pciutils", common) self.assertIn("hwdata", common) self.assertIn("kmod", common) - for a in ("x86_64", "aarch64", "ppc64le"): - self.assertIn("mstflint", arch.get(a, [])) - self.assertNotIn("s390x", arch) - - def test_subshell_arch_conditional_var_eq(self): - run_values = [ - 'SPECIAL=$(if [ "$(uname -m)" == "x86_64" ]; then echo -n intel-pkg ; fi) && ' - "yum -y install base-pkg $SPECIAL" - ] - common, arch = extract_packages_from_run_commands(run_values) - self.assertNotIn("intel-pkg", common) - self.assertIn("base-pkg", common) - self.assertEqual(arch, {"x86_64": ["intel-pkg"]}) - - def test_subshell_no_arch_condition(self): - run_values = ['EXTRA=$(echo extra-pkg) && yum -y install base-pkg $EXTRA'] - common, arch = extract_packages_from_run_commands(run_values) - self.assertIn("extra-pkg", common) - self.assertIn("base-pkg", common) self.assertEqual(arch, {}) - def test_subshell_arch_conditional_var_go_arch(self): - run_values = [ - 'SPECIAL=$(if [ "$(go env GOARCH)" == "amd64" ]; then echo -n x86-only ; fi) && ' - "yum -y install common-pkg $SPECIAL" - ] - common, arch = extract_packages_from_run_commands(run_values) - self.assertNotIn("x86-only", common) - self.assertIn("common-pkg", common) - self.assertEqual(arch, {"x86_64": ["x86-only"]}) - def test_if_else_var_with_nested_resolution(self): run_values = [ 'if yum install -y iotop >/dev/null 2>&1; then IOTOP_PKG="iotop"; ' @@ -604,17 +575,6 @@ def test_builddep_spec_file(self): _, _, _, _, builddep, _ = analyze_run_commands(run_values) self.assertEqual(builddep, ["mypackage.spec"]) - def test_build_dep_hyphenated(self): - run_values = ["dnf build-dep tuned.spec -y"] - _, _, _, _, builddep, _ = analyze_run_commands(run_values) - self.assertEqual(builddep, ["tuned.spec"]) - - def test_build_dep_hyphenated_with_install(self): - run_values = ["dnf install -y gcc rpm-build && cd assets/tuned/daemon && dnf build-dep tuned.spec -y"] - common, _, _, _, builddep, _ = analyze_run_commands(run_values) - self.assertIn("gcc", common) - self.assertEqual(builddep, ["tuned.spec"]) - class TestModuleParsing(unittest.TestCase): def test_module_install(self): @@ -642,25 +602,3 @@ def test_module_without_stream_ignored(self): run_values = ["dnf module enable -y nodejs"] _, _, _, _, _, modules = analyze_run_commands(run_values) self.assertEqual(modules, []) - - def test_variable_package_manager(self): - run_values = ["${DNF} install -y openssh-clients"] - pkgs, _, _, _, _, _ = analyze_run_commands(run_values, env_vars={"DNF": "microdnf"}) - self.assertEqual(pkgs, ["openssh-clients"]) - - def test_variable_package_manager_in_conditional(self): - run_values = [ - "if ! rpm -q openssh-clients; then ${DNF} install -y openssh-clients " - "&& ${DNF} clean all && rm -rf /var/cache/dnf/*; fi" - ] - pkgs, _, _, _, _, _ = analyze_run_commands(run_values, env_vars={"DNF": "microdnf"}) - self.assertEqual(pkgs, ["openssh-clients"]) - - def test_variable_package_manager_multiple_commands(self): - run_values = [ - "if ! rpm -q openssh-clients; then ${DNF} install -y openssh-clients && ${DNF} clean all; fi", - "if ! rpm -q libvirt-libs; then ${DNF} install -y libvirt-libs && ${DNF} clean all; fi", - "if ! command -v tar; then ${DNF} install -y tar && ${DNF} clean all; fi", - ] - pkgs, _, _, _, _, _ = analyze_run_commands(run_values, env_vars={"DNF": "microdnf"}) - self.assertEqual(pkgs, ["libvirt-libs", "openssh-clients", "tar"]) diff --git a/doozer/tests/test_assembly_inspector.py b/doozer/tests/test_assembly_inspector.py index ba5adc15b2..2684a2be08 100644 --- a/doozer/tests/test_assembly_inspector.py +++ b/doozer/tests/test_assembly_inspector.py @@ -79,27 +79,4 @@ def test_check_installed_rpms_in_image(self): issues = ai.check_installed_rpms_in_image("foo", build_inspector, None) self.assertEqual(issues, []) - def test_is_installed_rpm_in_tag(self): - rt = MagicMock(mode="both", group_config=Model({})) - brew_session = MagicMock() - ai = AssemblyInspector(rt, brew_session) - - # None installed_rpm returns False - self.assertFalse(ai._is_installed_rpm_in_tag(None, "my-candidate-tag")) - - # Installed RPM is tagged in the candidate tag - brew_session.listTags.return_value = [ - {"name": "my-candidate-tag"}, - {"name": "other-tag"}, - ] - installed_rpm = {"id": 100, "nvr": "cri-o-1.28.0-1.el9"} - self.assertTrue(ai._is_installed_rpm_in_tag(installed_rpm, "my-candidate-tag")) - - # Installed RPM is NOT tagged in the candidate tag - brew_session.listTags.return_value = [ - {"name": "rhel-9.2.0-z"}, - {"name": "RHSA-2026:13971-released"}, - ] - self.assertFalse(ai._is_installed_rpm_in_tag(installed_rpm, "my-candidate-tag")) - pass diff --git a/elliott/elliottlib/cli/process_release_from_fbc_bugs_cli.py b/elliott/elliottlib/cli/process_release_from_fbc_bugs_cli.py index 6e129c74e2..2968a30450 100644 --- a/elliott/elliottlib/cli/process_release_from_fbc_bugs_cli.py +++ b/elliott/elliottlib/cli/process_release_from_fbc_bugs_cli.py @@ -6,10 +6,9 @@ import click from artcommonlib.jira_config import JIRA_SERVER_URL from artcommonlib.util import new_roundtrip_yaml_handler -from doozerlib.util import konflux_application_name, konflux_image_component_name from elliottlib.bzutil import JIRABugTracker -from elliottlib.cli.attach_cve_flaws_cli import _image_uses_builder, get_konflux_component_by_component +from elliottlib.cli.attach_cve_flaws_cli import get_konflux_component_by_component from elliottlib.cli.common import cli, click_coroutine from elliottlib.runtime import Runtime from elliottlib.shipment_model import CveAssociation, ReleaseNotes @@ -19,10 +18,6 @@ YAML = new_roundtrip_yaml_handler() logger = logging.getLogger(__name__) -GOLANG_BUILDER_DELIVERY_REPO = "openshift4/openshift-golang-builder" -GOLANG_BUILDER_COMPONENT = "openshift-golang-builder-container" -GOLANG_BUILDER_PSCOMPONENTS = {GOLANG_BUILDER_DELIVERY_REPO, GOLANG_BUILDER_COMPONENT} - def _create_jira_tracker() -> JIRABugTracker: """Create a JIRABugTracker with minimal config, bypassing bug.yml from ocp-build-data. @@ -60,16 +55,6 @@ def _extract_pscomponent_from_labels(labels: List[str]) -> Optional[str]: return None -def _get_golang_builder_components(runtime) -> list[str]: - """Find all Konflux component names in the group that use the golang builder.""" - application = konflux_application_name(runtime.group) - components = [] - for image_meta in runtime.image_metas(): - if _image_uses_builder(image_meta, logger): - components.append(konflux_image_component_name(application, image_meta.distgit_key)) - return sorted(components) - - async def process_bugs(runtime: Runtime, jira_ids: List[str]) -> ReleaseNotes: """Process a list of JIRA IDs and produce a ReleaseNotes object. @@ -89,7 +74,6 @@ async def process_bugs(runtime: Runtime, jira_ids: List[str]) -> ReleaseNotes: cve_associations: list[CveAssociation] = [] flaw_bug_ids: list[int] = [] all_jira_ids: list[str] = [] - cve_mapping_errors: list[str] = [] for jira_id in jira_ids: jira_id = jira_id.strip() @@ -110,58 +94,32 @@ async def process_bugs(runtime: Runtime, jira_ids: List[str]) -> ReleaseNotes: cve_id = _extract_cve_id_from_labels(labels) if not cve_id: - msg = f"JIRA {jira_id} is Vulnerability but has no CVE label" - logger.error(msg) - cve_mapping_errors.append(msg) + logger.warning("JIRA %s is Vulnerability but has no CVE label, skipping CVE association", jira_id) continue pscomponent = _extract_pscomponent_from_labels(labels) if not pscomponent: - msg = f"JIRA {jira_id} (CVE {cve_id}) has no pscomponent label" - logger.error(msg) - cve_mapping_errors.append(msg) + logger.warning("JIRA %s (CVE %s) has no pscomponent label, skipping CVE association", jira_id, cve_id) continue distgit_component = get_component_by_delivery_repo(runtime, pscomponent) if not distgit_component: - if pscomponent in GOLANG_BUILDER_PSCOMPONENTS: - builder_components = _get_golang_builder_components(runtime) - if not builder_components: - msg = f"JIRA {jira_id} (CVE {cve_id}): golang builder CVE but no components use the builder" - logger.error(msg) - cve_mapping_errors.append(msg) - continue - logger.info( - "JIRA %s: CVE %s is a golang builder CVE, fanning out to %d components: %s", - jira_id, - cve_id, - len(builder_components), - builder_components, - ) - for comp in builder_components: - cve_associations.append(CveAssociation(key=cve_id, component=comp)) - else: - msg = f"JIRA {jira_id} (CVE {cve_id}): pscomponent '{pscomponent}' not found in delivery_repo_names" - logger.error(msg) - cve_mapping_errors.append(msg) - continue - - bug_flaw_ids = _extract_flaw_bug_ids_from_labels(labels) - if bug_flaw_ids: - logger.info("JIRA %s: found flaw bug IDs %s", jira_id, bug_flaw_ids) - flaw_bug_ids.extend(bug_flaw_ids) - else: - logger.warning("JIRA %s (CVE %s) has no flaw:bz# labels", jira_id, cve_id) + logger.warning( + "JIRA %s (CVE %s): pscomponent '%s' not found in delivery_repo_names, skipping", + jira_id, + cve_id, + pscomponent, + ) continue konflux_component = get_konflux_component_by_component(runtime, distgit_component) if not konflux_component: - msg = ( - f"JIRA {jira_id} (CVE {cve_id}): distgit component '{distgit_component}' " - f"could not be mapped to a Konflux component" + logger.warning( + "JIRA %s (CVE %s): distgit component '%s' could not be mapped to a Konflux component, skipping", + jira_id, + cve_id, + distgit_component, ) - logger.error(msg) - cve_mapping_errors.append(msg) continue logger.info( @@ -181,12 +139,6 @@ async def process_bugs(runtime: Runtime, jira_ids: List[str]) -> ReleaseNotes: else: logger.warning("JIRA %s (CVE %s) has no flaw:bz# labels", jira_id, cve_id) - if cve_mapping_errors: - raise RuntimeError( - f"Failed to map {len(cve_mapping_errors)} CVE(s) to components:\n" - + "\n".join(f" - {e}" for e in cve_mapping_errors) - ) - advisory_type = "RHSA" if cve_associations else "RHBA" logger.info("Advisory type determined: %s (CVEs found: %d)", advisory_type, len(cve_associations)) diff --git a/elliott/tests/test_process_release_from_fbc_bugs_cli.py b/elliott/tests/test_process_release_from_fbc_bugs_cli.py index 1642209fd0..81b04f5f25 100644 --- a/elliott/tests/test_process_release_from_fbc_bugs_cli.py +++ b/elliott/tests/test_process_release_from_fbc_bugs_cli.py @@ -3,7 +3,6 @@ from artcommonlib.jira_config import JIRA_DOMAIN_NAME from elliottlib.cli.process_release_from_fbc_bugs_cli import ( - GOLANG_BUILDER_DELIVERY_REPO, _extract_cve_id_from_labels, _extract_flaw_bug_ids_from_labels, _extract_pscomponent_from_labels, @@ -13,7 +12,6 @@ PATCH_CREATE_TRACKER = "elliottlib.cli.process_release_from_fbc_bugs_cli._create_jira_tracker" PATCH_GET_DELIVERY_REPO = "elliottlib.cli.process_release_from_fbc_bugs_cli.get_component_by_delivery_repo" PATCH_GET_KONFLUX_COMPONENT = "elliottlib.cli.process_release_from_fbc_bugs_cli.get_konflux_component_by_component" -PATCH_GET_BUILDER_COMPONENTS = "elliottlib.cli.process_release_from_fbc_bugs_cli._get_golang_builder_components" class TestLabelExtraction(unittest.TestCase): @@ -148,8 +146,8 @@ async def test_mixed_cve_and_regular_bugs(self, mock_create_tracker, mock_get_de @patch(PATCH_GET_KONFLUX_COMPONENT) @patch(PATCH_GET_DELIVERY_REPO) @patch(PATCH_CREATE_TRACKER) - async def test_cve_without_cve_label_raises(self, mock_create_tracker, mock_get_delivery, mock_get_konflux): - """A Vulnerability JIRA without a CVE label should raise RuntimeError.""" + async def test_cve_without_cve_label_skipped(self, mock_create_tracker, mock_get_delivery, mock_get_konflux): + """A Vulnerability JIRA without a CVE label should be skipped for CVE association but still listed.""" bug = self._make_jira_bug( "OADP-9999", is_vulnerability=True, @@ -160,17 +158,22 @@ async def test_cve_without_cve_label_raises(self, mock_create_tracker, mock_get_ mock_tracker.get_bug.return_value = bug mock_create_tracker.return_value = mock_tracker - with self.assertRaises(RuntimeError) as ctx: - await process_bugs(self.mock_runtime, ["OADP-9999"]) + result = await process_bugs(self.mock_runtime, ["OADP-9999"]) - self.assertIn("OADP-9999", str(ctx.exception)) - self.assertIn("no CVE label", str(ctx.exception)) + self.assertEqual(result.type, "RHBA") + self.assertIsNone(result.cves) + fixed_ids = [i.id for i in result.issues.fixed] + self.assertIn("OADP-9999", fixed_ids) + mock_get_delivery.assert_not_called() + mock_get_konflux.assert_not_called() @patch(PATCH_GET_KONFLUX_COMPONENT) @patch(PATCH_GET_DELIVERY_REPO) @patch(PATCH_CREATE_TRACKER) - async def test_cve_without_pscomponent_label_raises(self, mock_create_tracker, mock_get_delivery, mock_get_konflux): - """A Vulnerability with CVE label but no pscomponent label should raise RuntimeError.""" + async def test_cve_without_pscomponent_label_skipped( + self, mock_create_tracker, mock_get_delivery, mock_get_konflux + ): + """A Vulnerability with CVE label but no pscomponent label should skip CVE association.""" bug = self._make_jira_bug( "OADP-8888", is_vulnerability=True, @@ -181,17 +184,18 @@ async def test_cve_without_pscomponent_label_raises(self, mock_create_tracker, m mock_tracker.get_bug.return_value = bug mock_create_tracker.return_value = mock_tracker - with self.assertRaises(RuntimeError) as ctx: - await process_bugs(self.mock_runtime, ["OADP-8888"]) + result = await process_bugs(self.mock_runtime, ["OADP-8888"]) - self.assertIn("OADP-8888", str(ctx.exception)) - self.assertIn("no pscomponent label", str(ctx.exception)) + self.assertEqual(result.type, "RHBA") + self.assertIsNone(result.cves) + mock_get_delivery.assert_not_called() + mock_get_konflux.assert_not_called() @patch(PATCH_GET_KONFLUX_COMPONENT) @patch(PATCH_GET_DELIVERY_REPO) @patch(PATCH_CREATE_TRACKER) - async def test_unmapped_delivery_repo_raises(self, mock_create_tracker, mock_get_delivery, mock_get_konflux): - """When pscomponent can't be found in delivery_repo_names, should raise RuntimeError.""" + async def test_unmapped_delivery_repo_skipped(self, mock_create_tracker, mock_get_delivery, mock_get_konflux): + """When pscomponent can't be found in delivery_repo_names, CVE association is skipped.""" mock_get_delivery.return_value = None bug = self._make_jira_bug( @@ -204,17 +208,20 @@ async def test_unmapped_delivery_repo_raises(self, mock_create_tracker, mock_get mock_tracker.get_bug.return_value = bug mock_create_tracker.return_value = mock_tracker - with self.assertRaises(RuntimeError) as ctx: - await process_bugs(self.mock_runtime, ["OADP-7777"]) + result = await process_bugs(self.mock_runtime, ["OADP-7777"]) - self.assertIn("OADP-7777", str(ctx.exception)) - self.assertIn("unknown/unknown-container", str(ctx.exception)) + self.assertEqual(result.type, "RHBA") + self.assertIsNone(result.cves) + fixed_ids = [i.id for i in result.issues.fixed] + self.assertIn("OADP-7777", fixed_ids) + mock_get_delivery.assert_called_once_with(self.mock_runtime, "unknown/unknown-container") + mock_get_konflux.assert_not_called() @patch(PATCH_GET_KONFLUX_COMPONENT) @patch(PATCH_GET_DELIVERY_REPO) @patch(PATCH_CREATE_TRACKER) - async def test_unmapped_konflux_component_raises(self, mock_create_tracker, mock_get_delivery, mock_get_konflux): - """When distgit component can't be mapped to a Konflux component, should raise RuntimeError.""" + async def test_unmapped_konflux_component_skipped(self, mock_create_tracker, mock_get_delivery, mock_get_konflux): + """When distgit component can't be mapped to a Konflux component, CVE association is skipped.""" mock_get_delivery.return_value = "some-container" mock_get_konflux.return_value = None @@ -228,148 +235,14 @@ async def test_unmapped_konflux_component_raises(self, mock_create_tracker, mock mock_tracker.get_bug.return_value = bug mock_create_tracker.return_value = mock_tracker - with self.assertRaises(RuntimeError) as ctx: - await process_bugs(self.mock_runtime, ["OADP-7777"]) - - self.assertIn("OADP-7777", str(ctx.exception)) - self.assertIn("some-container", str(ctx.exception)) - - @patch(PATCH_GET_KONFLUX_COMPONENT) - @patch(PATCH_GET_DELIVERY_REPO) - @patch(PATCH_CREATE_TRACKER) - async def test_multiple_mapping_errors_all_reported(self, mock_create_tracker, mock_get_delivery, mock_get_konflux): - """All CVE mapping errors should be collected and reported in a single RuntimeError.""" - mock_get_delivery.return_value = None + result = await process_bugs(self.mock_runtime, ["OADP-7777"]) - bug1 = self._make_jira_bug( - "OADP-1111", - is_vulnerability=True, - labels=["CVE-2025-00001", "pscomponent:unknown/repo-a"], - ) - bug2 = self._make_jira_bug( - "OADP-2222", - is_vulnerability=True, - labels=["CVE-2025-00002", "pscomponent:unknown/repo-b"], - ) - - mock_tracker = Mock() - mock_tracker.get_bug.side_effect = lambda k: {"OADP-1111": bug1, "OADP-2222": bug2}[k] - mock_create_tracker.return_value = mock_tracker - - with self.assertRaises(RuntimeError) as ctx: - await process_bugs(self.mock_runtime, ["OADP-1111", "OADP-2222"]) - - error_msg = str(ctx.exception) - self.assertIn("2 CVE(s)", error_msg) - self.assertIn("OADP-1111", error_msg) - self.assertIn("OADP-2222", error_msg) - - @patch(PATCH_GET_BUILDER_COMPONENTS) - @patch(PATCH_GET_KONFLUX_COMPONENT) - @patch(PATCH_GET_DELIVERY_REPO) - @patch(PATCH_CREATE_TRACKER) - async def test_golang_builder_cve_fans_out( - self, mock_create_tracker, mock_get_delivery, mock_get_konflux, mock_get_builder - ): - """A golang builder CVE should fan out to all components using the builder.""" - mock_get_delivery.return_value = None - mock_get_builder.return_value = ["oadp-1-4-oadp-operator", "oadp-1-4-oadp-velero"] - - bug = self._make_jira_bug( - "OADP-7902", - is_vulnerability=True, - labels=[ - "CVE-2026-32281", - f"pscomponent:{GOLANG_BUILDER_DELIVERY_REPO}", - "flaw:bz#2456333", - ], - ) - - mock_tracker = Mock() - mock_tracker.get_bug.return_value = bug - mock_create_tracker.return_value = mock_tracker - - result = await process_bugs(self.mock_runtime, ["OADP-7902"]) - - self.assertEqual(result.type, "RHSA") - self.assertEqual(len(result.cves), 2) - cve_components = {c.component for c in result.cves} - self.assertEqual(cve_components, {"oadp-1-4-oadp-operator", "oadp-1-4-oadp-velero"}) - for cve in result.cves: - self.assertEqual(cve.key, "CVE-2026-32281") - - flaw_ids = {i.id for i in result.issues.fixed if i.source == "bugzilla.redhat.com"} - self.assertIn("2456333", flaw_ids) - - @patch(PATCH_GET_BUILDER_COMPONENTS) - @patch(PATCH_GET_KONFLUX_COMPONENT) - @patch(PATCH_GET_DELIVERY_REPO) - @patch(PATCH_CREATE_TRACKER) - async def test_golang_builder_cve_no_consumers_raises( - self, mock_create_tracker, mock_get_delivery, mock_get_konflux, mock_get_builder - ): - """A golang builder CVE with no builder-consuming components should raise RuntimeError.""" - mock_get_delivery.return_value = None - mock_get_builder.return_value = [] - - bug = self._make_jira_bug( - "OADP-7902", - is_vulnerability=True, - labels=["CVE-2026-32281", f"pscomponent:{GOLANG_BUILDER_DELIVERY_REPO}"], - ) - - mock_tracker = Mock() - mock_tracker.get_bug.return_value = bug - mock_create_tracker.return_value = mock_tracker - - with self.assertRaises(RuntimeError) as ctx: - await process_bugs(self.mock_runtime, ["OADP-7902"]) - - self.assertIn("OADP-7902", str(ctx.exception)) - self.assertIn("no components use the builder", str(ctx.exception)) - - @patch(PATCH_GET_BUILDER_COMPONENTS) - @patch(PATCH_GET_KONFLUX_COMPONENT) - @patch(PATCH_GET_DELIVERY_REPO) - @patch(PATCH_CREATE_TRACKER) - async def test_golang_builder_cve_mixed_with_regular( - self, mock_create_tracker, mock_get_delivery, mock_get_konflux, mock_get_builder - ): - """A golang builder CVE and a regular CVE should both produce associations.""" - mock_get_delivery.side_effect = lambda _rt, repo: { - "oadp/oadp-velero-rhel9": "oadp-velero-container", - }.get(repo) - mock_get_konflux.side_effect = lambda _rt, comp: { - "oadp-velero-container": "oadp-1-4-oadp-velero", - }.get(comp) - mock_get_builder.return_value = ["oadp-1-4-oadp-operator", "oadp-1-4-oadp-velero"] - - builder_bug = self._make_jira_bug( - "OADP-7902", - is_vulnerability=True, - labels=["CVE-2026-32281", f"pscomponent:{GOLANG_BUILDER_DELIVERY_REPO}", "flaw:bz#2456333"], - ) - regular_cve = self._make_jira_bug( - "OADP-7792", - is_vulnerability=True, - labels=["CVE-2026-32280", "pscomponent:oadp/oadp-velero-rhel9", "flaw:bz#2456339"], - ) - - mock_tracker = Mock() - mock_tracker.get_bug.side_effect = lambda k: {"OADP-7902": builder_bug, "OADP-7792": regular_cve}[k] - mock_create_tracker.return_value = mock_tracker - - result = await process_bugs(self.mock_runtime, ["OADP-7902", "OADP-7792"]) - - self.assertEqual(result.type, "RHSA") - self.assertEqual(len(result.cves), 3) - - builder_cves = [c for c in result.cves if c.key == "CVE-2026-32281"] - self.assertEqual(len(builder_cves), 2) - - regular_cves = [c for c in result.cves if c.key == "CVE-2026-32280"] - self.assertEqual(len(regular_cves), 1) - self.assertEqual(regular_cves[0].component, "oadp-1-4-oadp-velero") + self.assertEqual(result.type, "RHBA") + self.assertIsNone(result.cves) + fixed_ids = [i.id for i in result.issues.fixed] + self.assertIn("OADP-7777", fixed_ids) + mock_get_delivery.assert_called_once_with(self.mock_runtime, "oadp/some-rhel9") + mock_get_konflux.assert_called_once_with(self.mock_runtime, "some-container") @patch(PATCH_GET_KONFLUX_COMPONENT) @patch(PATCH_GET_DELIVERY_REPO) diff --git a/ocp-build-data-validator/tests/test_schema/test_group_schema.py b/ocp-build-data-validator/tests/test_schema/test_group_schema.py index a368d6b5ca..cc2833243d 100644 --- a/ocp-build-data-validator/tests/test_schema/test_group_schema.py +++ b/ocp-build-data-validator/tests/test_schema/test_group_schema.py @@ -26,29 +26,6 @@ def test_validate_with_invalid_bridge_release_config(self): } self.assertIn("'yes' is not of type 'boolean'", group_schema.validate("group.yml", invalid_data)) - def test_validate_reposync_requires_enabled(self): - data_missing_enabled = { - "repos": { - "my-repo": { - "conf": {"baseurl": {"x86_64": "https://example.com/repo/"}}, - "reposync": {"latest_only": False}, - } - } - } - result = group_schema.validate("group.yml", data_missing_enabled) - self.assertIn("'enabled' is a required property", result) - - def test_validate_reposync_with_enabled(self): - data_with_enabled = { - "repos": { - "my-repo": { - "conf": {"baseurl": {"x86_64": "https://example.com/repo/"}}, - "reposync": {"enabled": False}, - } - } - } - self.assertEqual("", group_schema.validate("group.yml", data_with_enabled)) - def test_validate_with_mismatched_bridge_release_basis_group(self): invalid_data = { "name": "openshift-4.23", diff --git a/ocp-build-data-validator/validator/json_schemas/repos.schema.json b/ocp-build-data-validator/validator/json_schemas/repos.schema.json index 07fdbbc4b3..ad7857b528 100644 --- a/ocp-build-data-validator/validator/json_schemas/repos.schema.json +++ b/ocp-build-data-validator/validator/json_schemas/repos.schema.json @@ -47,7 +47,6 @@ "type": "boolean" } }, - "required": ["enabled"], "additionalProperties": false }, "reposync!": { diff --git a/pyartcd/pyartcd/__main__.py b/pyartcd/pyartcd/__main__.py index 6e8cb58140..88ea7717a7 100644 --- a/pyartcd/pyartcd/__main__.py +++ b/pyartcd/pyartcd/__main__.py @@ -21,6 +21,7 @@ fbc_import_from_index, gen_assembly, gen_assembly_lp, + golang_builder_shipment, images_health, layered_products_scan_konflux, ocp, @@ -83,6 +84,7 @@ "fbc_import_from_index", "gen_assembly", "gen_assembly_lp", + "golang_builder_shipment", "images_health", "layered_products_scan_konflux", "ocp", diff --git a/pyartcd/pyartcd/pipelines/__init__.py b/pyartcd/pyartcd/pipelines/__init__.py index 3d801037b7..89619d36ef 100644 --- a/pyartcd/pyartcd/pipelines/__init__.py +++ b/pyartcd/pyartcd/pipelines/__init__.py @@ -20,6 +20,7 @@ fbc_import_from_index, gen_assembly, gen_assembly_targeted, + golang_builder_shipment, images_health, layered_products_scan_konflux, ocp, @@ -69,6 +70,7 @@ 'fbc_import_from_index', 'gen_assembly', 'gen_assembly_targeted', + 'golang_builder_shipment', 'images_health', 'layered_products_scan_konflux', 'ocp', diff --git a/pyartcd/pyartcd/pipelines/build_microshift_bootc.py b/pyartcd/pyartcd/pipelines/build_microshift_bootc.py index a6c9f22948..d6126ea800 100644 --- a/pyartcd/pyartcd/pipelines/build_microshift_bootc.py +++ b/pyartcd/pyartcd/pipelines/build_microshift_bootc.py @@ -509,7 +509,7 @@ async def _rebuild_needed(): self._logger.info("Skipping plashet sync for %s", microshift_plashet_name) return - result = jenkins.start_build_plashets( + jenkins.start_build_plashets( group=self.group, release=default_release_suffix(), assembly=self.assembly, @@ -519,11 +519,6 @@ async def _rebuild_needed(): dry_run=self.runtime.dry_run, block_until_complete=True, ) - if result != "SUCCESS": - raise RuntimeError( - f"build-plashets job for {microshift_plashet_name} failed with result: {result}. " - f"Plashet must be built successfully before proceeding with the bootc image build." - ) async def _get_microshift_rpm_commit(self) -> str: """ diff --git a/pyartcd/pyartcd/pipelines/golang_builder_shipment.py b/pyartcd/pyartcd/pipelines/golang_builder_shipment.py new file mode 100644 index 0000000000..2e248e7ead --- /dev/null +++ b/pyartcd/pyartcd/pipelines/golang_builder_shipment.py @@ -0,0 +1,519 @@ +""" +Pipeline to create shipment MRs in ocp-shipment-data for golang builder releases. + +Invocation:: + + artcd golang-builder-shipment \\ + --ocp-version 4.22 \\ + --golang-nvrs golang-1.25.9-1.el9 + + # or with explicit Konflux image NVRs: + artcd golang-builder-shipment \\ + --ocp-version 4.22 \\ + --golang-group rhel-9-golang-1.25 \\ + openshift-golang-builder-container-v1.25.9-202605121249.p2.gdf787b0.el9 + +This pipeline: +1. Derives golang_group from the NVRs (or accepts it explicitly) +2. Resolves Konflux image NVRs from golang RPM NVRs if needed +3. Reads software_lifecycle.phase from ocp-build-data to pick the correct ReleasePlan +4. Builds a ShipmentConfig YAML with snapshot from the resolved NVRs +5. Creates a draft MR in ocp-shipment-data +""" + +import logging +import os +import re +import shutil +import tempfile +from datetime import datetime, timezone +from io import StringIO +from pathlib import Path +from typing import List, Optional, Tuple +from urllib.parse import urlparse + +import aiohttp +import click +import gitlab as python_gitlab +from artcommonlib import exectools +from artcommonlib.constants import GOLANG_BUILDER_IMAGE_NAME, SHIPMENT_DATA_URL_TEMPLATE +from artcommonlib.konflux.konflux_build_record import ArtifactType, Engine, KonfluxBuildOutcome +from artcommonlib.konflux.konflux_db import KonfluxDb +from artcommonlib.release_util import SoftwareLifecyclePhase, isolate_el_version_in_release +from artcommonlib.rpm_utils import parse_nvr +from artcommonlib.util import new_roundtrip_yaml_handler +from doozerlib.constants import ART_IMAGES_BASE_APPLICATION +from elliottlib.constants import GOLANG_BUILDER_CVE_COMPONENT +from elliottlib.shipment_model import ( + Data, + Environments, + Metadata, + ReleaseNotes, + Shipment, + ShipmentConfig, + ShipmentEnv, + Snapshot, + SnapshotSpec, +) + +from pyartcd import constants +from pyartcd.cli import cli, click_coroutine, pass_runtime +from pyartcd.git import GitRepository +from pyartcd.runtime import Runtime + +_LOGGER = logging.getLogger(__name__) +yaml = new_roundtrip_yaml_handler() + +GOLANG_BUILDER_SHIPMENT_RELEASE_PLAN_MAP = { + "prod": "ocp-art-golang-builder-prod-rhel9", + "ec": "ocp-art-golang-builder-ec-rhel9", +} + + +def derive_golang_group(nvrs: List[str]) -> str: + """Derive the ocp-build-data golang group from NVR patterns. + + Supports both golang RPM NVRs (golang-1.25.9-1.el9) and + Konflux image NVRs (openshift-golang-builder-container-v1.25.9-...). + """ + for nvr in nvrs: + # Konflux image NVR: openshift-golang-builder-container-v1.25.9-...el9 + m = re.search(r"v(\d+)\.(\d+)\.\d+.*\.el(\d+)", nvr) + if m: + return f"rhel-{m.group(3)}-golang-{m.group(1)}.{m.group(2)}" + + # RPM NVR: golang-1.25.9-1.el9 + parsed = parse_nvr(nvr) + if parsed["name"] == "golang": + major_minor = ".".join(parsed["version"].split(".")[:2]) + el_v = isolate_el_version_in_release(parsed["release"]) + if el_v is not None: + return f"rhel-{el_v}-golang-{major_minor}" + + raise ValueError(f"Cannot derive golang group from NVRs: {nvrs}") + + +async def resolve_konflux_image_nvrs(golang_nvrs: List[str]) -> List[str]: + """Resolve golang RPM NVRs to Konflux golang-builder image NVRs via Konflux DB.""" + image_nvrs = [] + db = KonfluxDb() + + for nvr in golang_nvrs: + parsed = parse_nvr(nvr) + go_version = parsed["version"] + el_v = isolate_el_version_in_release(parsed["release"]) + if el_v is None: + raise ValueError(f"Cannot detect RHEL version from NVR: {nvr}") + + extra_patterns = {"nvr": f"{GOLANG_BUILDER_CVE_COMPONENT}-v{go_version}"} + record = await anext( + db.search_builds_by_fields( + where={ + "name": GOLANG_BUILDER_IMAGE_NAME, + "el_target": f"el{el_v}", + "artifact_type": str(ArtifactType.IMAGE), + "outcome": str(KonfluxBuildOutcome.SUCCESS), + "engine": str(Engine.KONFLUX), + }, + extra_patterns=extra_patterns, + limit=1, + ), + None, + ) + if not record: + raise RuntimeError( + f"No Konflux golang-builder image found for go {go_version} el{el_v}. " + f"Has update-golang built one for {nvr}?" + ) + _LOGGER.info("Resolved %s → %s", nvr, record.nvr) + image_nvrs.append(record.nvr) + + return image_nvrs + + +async def resolve_lifecycle_env(ocp_version: str, data_path: Optional[str] = None) -> str: + """Determine 'prod' or 'ec' by reading software_lifecycle.phase from ocp-build-data. + + Reads group.yml from the openshift-{ocp_version} branch. If the phase is + ``pre-release``, returns ``'ec'``; otherwise returns ``'prod'``. + """ + base_url = (data_path or constants.OCP_BUILD_DATA_URL).rstrip("/") + branch = f"openshift-{ocp_version}" + + # GitHub raw URL — works for both github.com and GHE + if "github.com" in base_url: + raw_url = base_url.replace("github.com", "raw.githubusercontent.com") + f"/{branch}/group.yml" + else: + raw_url = f"{base_url}/raw/{branch}/group.yml" + + _LOGGER.info("Fetching lifecycle phase from %s", raw_url) + async with aiohttp.ClientSession() as session: + async with session.get(raw_url, timeout=aiohttp.ClientTimeout(total=30)) as resp: + if resp.status != 200: + raise RuntimeError( + f"Failed to fetch group.yml from {raw_url} (HTTP {resp.status}). " + f"Does the branch '{branch}' exist in ocp-build-data?" + ) + content = await resp.text() + + group_config = yaml.load(content) + phase_str = None + if group_config and isinstance(group_config, dict): + lifecycle = group_config.get("software_lifecycle") + if lifecycle and isinstance(lifecycle, dict): + phase_str = lifecycle.get("phase") + + if not phase_str: + _LOGGER.warning("No software_lifecycle.phase in group.yml for %s; defaulting to prod", branch) + return "prod" + + try: + phase = SoftwareLifecyclePhase.from_name(phase_str) + except ValueError: + _LOGGER.warning("Unknown lifecycle phase '%s' for %s; defaulting to prod", phase_str, branch) + return "prod" + + if phase == SoftwareLifecyclePhase.PRE_RELEASE: + _LOGGER.info("OCP %s is pre-release → using ec ReleasePlan", ocp_version) + return "ec" + + _LOGGER.info("OCP %s is %s → using prod ReleasePlan", ocp_version, phase_str) + return "prod" + + +class GolangBuilderShipmentPipeline: + """Creates a shipment MR in ocp-shipment-data for golang builder images.""" + + def __init__( + self, + runtime: Runtime, + ocp_version: str, + nvrs: List[str], + golang_group: Optional[str] = None, + art_jira: str = "", + shipment_data_repo_url: Optional[str] = None, + data_path: Optional[str] = None, + ): + self.runtime = runtime + self.ocp_version = ocp_version + self.golang_group = golang_group or derive_golang_group(nvrs) + self.nvrs = sorted(nvrs) + self.art_jira = art_jira + self.dry_run = runtime.dry_run + self.data_path = data_path + + self.product = "ocp" + self.gitlab_url = runtime.config.get("gitlab_url", "https://gitlab.cee.redhat.com") + self.working_dir = runtime.working_dir.absolute() + self._shipment_data_repo_dir = self.working_dir / "shipment-data-push" + + self.shipment_data_repo_pull_url = ( + shipment_data_repo_url + or runtime.config.get("shipment_config", {}).get("shipment_data_url") + or SHIPMENT_DATA_URL_TEMPLATE + ) + self.shipment_data_repo_push_url = ( + runtime.config.get("shipment_config", {}).get("shipment_data_push_url") or SHIPMENT_DATA_URL_TEMPLATE + ) + self.shipment_data_repo = GitRepository(self._shipment_data_repo_dir, self.dry_run) + + # Resolved lazily in run() via resolve_lifecycle_env + self.env: Optional[str] = None + self.release_plan: Optional[str] = None + + @staticmethod + def resolve_release_plan(env: str) -> str: + """Map lifecycle env to the correct ReleasePlan name.""" + plan = GOLANG_BUILDER_SHIPMENT_RELEASE_PLAN_MAP.get(env) + if not plan: + raise ValueError( + f"Unknown env '{env}'. Must be one of: {list(GOLANG_BUILDER_SHIPMENT_RELEASE_PLAN_MAP.keys())}" + ) + return plan + + @staticmethod + def basic_auth_url(url: str, token: str) -> str: + """Inject token into a GitLab URL for push authentication.""" + parsed = urlparse(url) + return f"{parsed.scheme}://oauth2:{token}@{parsed.hostname}{parsed.path}" + + async def run(self) -> str: + """Execute the pipeline end to end. + + Returns: + The URL of the created shipment MR. + """ + self.env = await resolve_lifecycle_env(self.ocp_version, self.data_path) + self.release_plan = self.resolve_release_plan(self.env) + + _LOGGER.info( + "Starting golang-builder-shipment pipeline: ocp_version=%s golang_group=%s env=%s release_plan=%s nvrs=%s", + self.ocp_version, + self.golang_group, + self.env, + self.release_plan, + self.nvrs, + ) + + self.setup_working_dir() + await self.setup_repos() + + shipment_config = await self.build_shipment_config() + mr_url = await self.create_shipment_mr(shipment_config) + + _LOGGER.info("Shipment MR created: %s", mr_url) + return mr_url + + def setup_working_dir(self) -> None: + self.working_dir.mkdir(parents=True, exist_ok=True) + if self._shipment_data_repo_dir.exists(): + shutil.rmtree(self._shipment_data_repo_dir, ignore_errors=True) + + async def setup_repos(self): + self._gitlab_token = os.getenv("GITLAB_TOKEN") + if not self._gitlab_token: + raise ValueError("GITLAB_TOKEN environment variable is required") + + await self.shipment_data_repo.setup( + remote_url=self.basic_auth_url(self.shipment_data_repo_push_url, self._gitlab_token), + upstream_remote_url=self.shipment_data_repo_pull_url, + ) + await self.shipment_data_repo.fetch_switch_branch("main") + + async def build_shipment_config(self) -> ShipmentConfig: + """Construct a ShipmentConfig from NVRs using elliott snapshot new.""" + + snapshot = await self._create_snapshot() + + # Override application to match RP/RPA configuration. elliott infers the + # Konflux application from the build pipeline URL (e.g. "rhel-9-golang-1-25"), + # but our ReleasePlan/RPA pair is registered under "art-images-base". + snapshot.spec.application = ART_IMAGES_BASE_APPLICATION + + metadata = Metadata( + product=self.product, + application=snapshot.spec.application, + group=self.golang_group, + assembly="stream", + ) + + # Stage and prod use the same ReleasePlan for golang builders — the + # RPA is per-lifecycle (ec vs prod), not per-environment. + environments = Environments( + stage=ShipmentEnv(releasePlan=self.release_plan), + prod=ShipmentEnv(releasePlan=self.release_plan), + ) + + release_notes = ReleaseNotes( + type="RHBA", + synopsis=f"Golang builder image update for OpenShift {self.ocp_version}", + topic=( + f"An update for the golang builder images is now available for " + f"Red Hat OpenShift Container Platform {self.ocp_version}." + ), + description=( + f"This update provides rebuilt golang builder images for " + f"Red Hat OpenShift Container Platform {self.ocp_version}.\n\n" + f"Golang group: {self.golang_group}" + ), + solution="The golang builder images are available from registry.redhat.io/openshift/golang-builder.", + ) + if self.art_jira: + release_notes.references = [f"https://redhat.atlassian.net/browse/{self.art_jira}"] + + shipment = Shipment( + metadata=metadata, + environments=environments, + snapshot=snapshot, + data=Data(releaseNotes=release_notes), + ) + + config = ShipmentConfig(shipment=shipment) + _LOGGER.info("Built ShipmentConfig with %d NVRs", len(self.nvrs)) + return config + + async def _create_snapshot(self) -> Snapshot: + """Create a Snapshot from NVRs using ``elliott snapshot new --builds-file``.""" + + with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f: + for nvr in self.nvrs: + f.write(nvr + "\n") + builds_file = f.name + + try: + cmd = [ + "elliott", + "--group", + self.golang_group, + "--assembly", + "stream", + "snapshot", + "new", + f"--builds-file={builds_file}", + ] + quay_auth_file = os.getenv("QUAY_AUTH_FILE") + if quay_auth_file: + cmd.append(f"--pull-secret={quay_auth_file}") + + rc, stdout, stderr = await exectools.cmd_gather_async(cmd, stderr=None, check=False) + if rc != 0: + raise RuntimeError(f"elliott snapshot new failed (rc={rc}): {stderr or stdout}") + if stdout: + _LOGGER.info("elliott snapshot new output:\n%s", stdout) + finally: + os.unlink(builds_file) + + snapshot_obj = yaml.load(stdout) + if not snapshot_obj or not isinstance(snapshot_obj, dict): + raise ValueError(f"elliott snapshot new returned invalid output: {stdout!r}") + + spec = snapshot_obj.get("spec") + if not spec: + raise ValueError(f"elliott snapshot new output missing 'spec': {snapshot_obj}") + + return Snapshot( + spec=SnapshotSpec(**spec), + nvrs=self.nvrs, + ) + + async def create_shipment_mr(self, shipment_config: ShipmentConfig) -> str: + """Write the shipment YAML and open a draft MR in ocp-shipment-data.""" + + timestamp = datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S") + source_branch = f"golang-builder-shipment-{self.golang_group}-{timestamp}" + + await self.shipment_data_repo.create_branch(source_branch) + + # Persist the YAML + application = shipment_config.shipment.metadata.application + relative_target_dir = Path("shipment") / self.product / self.golang_group / application / self.env + target_dir = self.shipment_data_repo._directory / relative_target_dir + target_dir.mkdir(parents=True, exist_ok=True) + + filename = f"stream.image.{timestamp}.yaml" + filepath = relative_target_dir / filename + shipment_dump = shipment_config.model_dump(exclude_unset=True, exclude_none=True) + out = StringIO() + yaml.dump(shipment_dump, out) + await self.shipment_data_repo.write_file(filepath, out.getvalue()) + await self.shipment_data_repo.add_all() + await self.shipment_data_repo.log_diff() + + commit_message = f"Add golang builder shipment for {self.golang_group}" + if self.art_jira: + commit_message += f"\n\nRef: {self.art_jira}" + job_url = os.getenv("BUILD_URL", "") + if job_url: + commit_message += f"\n{job_url}" + + pushed = await self.shipment_data_repo.commit_push(commit_message, safe=True) + if not pushed: + raise RuntimeError("Failed to push shipment data to remote") + + mr_title = f"Draft: Golang builder shipment for {self.golang_group}" + mr_description = f"Golang builder shipment for OCP {self.ocp_version}\n\n" + mr_description += f"Group: {self.golang_group}\n" + mr_description += f"Environment: {self.env}\n" + mr_description += f"ReleasePlan: {self.release_plan}\n" + mr_description += f"NVRs: {len(self.nvrs)}\n" + if self.art_jira: + mr_description += f"\nRef: https://redhat.atlassian.net/browse/{self.art_jira}" + if job_url: + mr_description += f"\nCreated by: {job_url}" + + if self.dry_run: + _LOGGER.info("[DRY-RUN] Would create MR: %s", mr_title) + return f"{self.gitlab_url}/placeholder/-/merge_requests/placeholder" + + gl = python_gitlab.Gitlab(self.gitlab_url, private_token=self._gitlab_token) + + def _get_project(url): + parsed = urlparse(url) + project_path = parsed.path.strip("/").removesuffix(".git") + return gl.projects.get(project_path) + + source_project = _get_project(self.shipment_data_repo_push_url) + target_project = _get_project(self.shipment_data_repo_pull_url) + + mr = source_project.mergerequests.create( + { + "source_branch": source_branch, + "target_project_id": target_project.id, + "target_branch": "main", + "title": mr_title, + "description": mr_description, + "remove_source_branch": True, + } + ) + _LOGGER.info("Created Draft MR: %s", mr.web_url) + return mr.web_url + + +@cli.command("golang-builder-shipment") +@click.option("--ocp-version", required=True, help="OCP version (e.g. 4.22)") +@click.option( + "--golang-group", + required=False, + default=None, + help="Golang builder group (e.g. rhel-9-golang-1.25). Derived from NVRs if omitted.", +) +@click.option( + "--golang-nvrs", + required=False, + default=None, + help="Golang RPM NVRs (comma-separated). Resolves to Konflux image NVRs automatically.", +) +@click.option("--art-jira", default="", help="Related ART Jira ticket (e.g. ART-20930)") +@click.option("--shipment-data-repo-url", default=None, help="Override ocp-shipment-data repo URL") +@click.option( + "--data-path", + required=False, + default=constants.OCP_BUILD_DATA_URL, + help="ocp-build-data URL (used to read software_lifecycle.phase)", +) +@click.argument("nvrs", nargs=-1, required=False) +@pass_runtime +@click_coroutine +async def golang_builder_shipment( + runtime: Runtime, + ocp_version: str, + golang_group: Optional[str], + golang_nvrs: Optional[str], + art_jira: str, + shipment_data_repo_url: Optional[str], + data_path: str, + nvrs: Tuple[str, ...], +): + """Create a shipment MR in ocp-shipment-data for golang builder images. + + Builds a ShipmentConfig YAML from the provided NVRs, auto-detects whether + to use the prod or ec ReleasePlan from ocp-build-data software_lifecycle.phase, + and opens a draft MR in ocp-shipment-data for ERT approval. + + Accepts either explicit Konflux image NVRs as positional arguments, or + --golang-nvrs with golang RPM NVRs (auto-resolved to Konflux image NVRs). + The --golang-group is derived from the NVRs when not specified. + """ + resolved_nvrs: List[str] = list(nvrs) + + if not resolved_nvrs and golang_nvrs: + rpm_nvrs = [n.strip() for n in golang_nvrs.replace(",", " ").split() if n.strip()] + _LOGGER.info("Resolving golang RPM NVRs to Konflux image NVRs: %s", rpm_nvrs) + resolved_nvrs = await resolve_konflux_image_nvrs(rpm_nvrs) + if not golang_group: + golang_group = derive_golang_group(rpm_nvrs) + + if not resolved_nvrs: + raise click.UsageError("Provide Konflux image NVRs as arguments or --golang-nvrs with golang RPM NVRs") + + pipeline = GolangBuilderShipmentPipeline( + runtime=runtime, + ocp_version=ocp_version, + nvrs=resolved_nvrs, + golang_group=golang_group, + art_jira=art_jira, + shipment_data_repo_url=shipment_data_repo_url, + data_path=data_path, + ) + mr_url = await pipeline.run() + click.echo(f"Shipment MR: {mr_url}") diff --git a/pyartcd/pyartcd/pipelines/okd.py b/pyartcd/pyartcd/pipelines/okd.py index 56f05f0106..3b0d717384 100644 --- a/pyartcd/pyartcd/pipelines/okd.py +++ b/pyartcd/pyartcd/pipelines/okd.py @@ -28,7 +28,7 @@ reset_fail_counter, ) -OKD_ARCHES = ['x86_64', 'aarch64'] +OKD_ARCHES = ['x86_64'] class BuildPlan: diff --git a/pyartcd/pyartcd/pipelines/quay_doomsday_backup.py b/pyartcd/pyartcd/pipelines/quay_doomsday_backup.py index f8b602b2cf..f256c8d285 100644 --- a/pyartcd/pyartcd/pipelines/quay_doomsday_backup.py +++ b/pyartcd/pyartcd/pipelines/quay_doomsday_backup.py @@ -99,14 +99,13 @@ async def run(self) -> None: self.runtime.logger.info("[DRY RUN] Would have messaged Slack") # Synchronize individual arches sequentially to help with quay returning 502 - failed_arches = [] + results = [] for arch in self.arches: - if not await self.sync_arch(arch): - failed_arches.append(arch) + results.append(await self.sync_arch(arch)) # Report the results to Slack if not self.runtime.dry_run: - if not failed_arches: + if all(results): await self.slack_client._client.reactions_add( channel=slack_channel_id, timestamp=main_message_ts, name="done_it_is" ) @@ -115,9 +114,6 @@ async def run(self) -> None: else: self.runtime.logger.info("[DRY RUN] Would have messaged Slack") - if failed_arches: - raise RuntimeError(f"Failed to sync arches for {self.version}: {', '.join(failed_arches)}") - @cli.command( "quay-doomsday-backup", diff --git a/pyartcd/pyartcd/pipelines/sync_ci_images.py b/pyartcd/pyartcd/pipelines/sync_ci_images.py index cf908efe8c..1f8c7653e8 100644 --- a/pyartcd/pyartcd/pipelines/sync_ci_images.py +++ b/pyartcd/pyartcd/pipelines/sync_ci_images.py @@ -53,7 +53,6 @@ def __init__( data_path: str = "", data_gitref: str = "", only_stream: str = "", - images: str = "", add_labels: str = "", assembly: str = "stream", skip_prs: bool = False, @@ -70,7 +69,6 @@ def __init__( data_path: ocp-build-data fork URL (default: official repo) data_gitref: ocp-build-data git branch/tag/sha (default: use version branch) only_stream: Specific stream from streams.yml. - images: Comma-separated distgit keys of images with ci_alignment.upstream_image. add_labels: Space-delimited labels to add to PRs assembly: Assembly name (default: "stream") skip_prs: Skip opening reconciliation PRs @@ -84,7 +82,6 @@ def __init__( self.data_path = data_path or OCP_BUILD_DATA_URL self.data_gitref = data_gitref self.only_stream = only_stream - self.images = [i.strip() for i in images.split(',') if i.strip()] if images else [] self.add_labels = add_labels self.assembly = assembly self.skip_prs = skip_prs @@ -109,9 +106,9 @@ def _validate_parameters(self) -> None: if not re.match(r'^\d+\.\d+$', self.version): raise ValueError(f"Invalid FOR_RELEASE format: {self.version}. Expected format: X.Y (e.g., 4.17)") - # Auto-set SKIP_PRS if ONLY_STREAM or IMAGES is set. - if (self.only_stream or self.images) and not self.skip_prs: - self._logger.info("Setting SKIP_PRS to true because ONLY_STREAM or IMAGES is set") + # Auto-set SKIP_PRS if ONLY_STREAM is set. + if self.only_stream and not self.skip_prs: + self._logger.info("Setting SKIP_PRS to true because ONLY_STREAM is set") self.skip_prs = True # Validate assembly format (alphanumeric, dash, dot, underscore) @@ -514,37 +511,24 @@ def _build_doozer_options(self, group_dir: Path, auth_file: str) -> str: f"--build-system {self.BUILD_SYSTEM} " f"--registry-config {auth_file}" ) + if self.only_stream: + doozer_opts += f" --only-streams {self.only_stream}" return doozer_opts - @property - def _stream_arg(self) -> str: - """Return the --stream subcommand arg if only_stream is set, empty string otherwise.""" - return f"--stream {self.only_stream}" if self.only_stream else "" - - @property - def _image_args(self) -> str: - """Return --image args for each image distgit key, empty string if none.""" - return " ".join(f"--image {img}" for img in self.images) if self.images else "" - - @property - def _filter_args(self) -> str: - """Return combined --stream and --image subcommand args.""" - return f"{self._stream_arg} {self._image_args}".strip() - async def _generate_and_apply_buildconfigs(self, doozer_opts: str) -> None: """Generate BuildConfigs and apply them to CI cluster.""" self._logger.info(f"{self.version}: Generating BuildConfigs") apply_flag = "" if self.runtime.dry_run else "--apply" await self._run_doozer_command( - doozer_opts, - "images:streams gen-buildconfigs", - f"{self._filter_args} -o {self._working_dir}/buildconfigs.yaml {apply_flag}", + doozer_opts, "images:streams gen-buildconfigs", f"-o {self._working_dir}/buildconfigs.yaml {apply_flag}" ) async def _mirror_images_to_ci(self, doozer_opts: str, auth_file: str) -> None: """Mirror builder and base images to CI registries.""" self._logger.info(f"{self.version}: Mirroring images") - mirror_args = f"{self._filter_args} --registry-auth {auth_file} " + # Pass --registry-auth at command level to work around doozer bug where + # global --registry-config doesn't get passed to oc image mirror commands + mirror_args = f"--registry-auth {auth_file} " if self.update_images_only_when_missing: mirror_args += "--only-if-missing " if self.runtime.dry_run: @@ -554,7 +538,8 @@ async def _mirror_images_to_ci(self, doozer_opts: str, auth_file: str) -> None: async def _trigger_ci_builds(self, doozer_opts: str, auth_file: str) -> None: """Start CI builds for updated images.""" self._logger.info(f"{self.version}: Starting builds") - start_builds_args = f"{self._filter_args} --registry-auth {auth_file} " + # Pass --registry-auth for image info and mirror operations in pre-build steps + start_builds_args = f"--registry-auth {auth_file} " if self.runtime.dry_run: start_builds_args += "--dry-run" await self._run_doozer_command(doozer_opts, "images:streams start-builds", start_builds_args.strip()) @@ -562,9 +547,7 @@ async def _trigger_ci_builds(self, doozer_opts: str, auth_file: str) -> None: async def _verify_upstream_consistency(self, doozer_opts: str, auth_file: str) -> None: """Verify CI imagestreams match expected state.""" self._logger.info(f"{self.version}: Checking upstream consistency") - await self._run_doozer_command( - doozer_opts, "images:streams check-upstream", f"{self._filter_args} --registry-auth {auth_file}" - ) + await self._run_doozer_command(doozer_opts, "images:streams check-upstream", f"--registry-auth {auth_file}") async def _open_reconciliation_prs(self, doozer_opts: str) -> int: """Open PRs to reconcile BuildConfig drift.""" @@ -701,12 +684,6 @@ async def run(self) -> int: default='', help='Process only specific stream from streams.yml. Automatically sets SKIP_PRS=true.', ) -@click.option( - '--images', - default='', - help='Comma-separated distgit keys to sync (e.g. ci-openshift-base.rhel10). ' - 'Each must have ci_alignment.upstream_image set. Automatically sets SKIP_PRS=true.', -) @click.option( '--add-labels', default='', help='Space-delimited labels to add to reconciliation PRs (e.g., "backport candidate")' ) @@ -734,7 +711,6 @@ async def sync_ci_images_cli( data_path: str, data_gitref: str, only_stream: str, - images: str, add_labels: str, assembly: str, skip_prs: bool, @@ -764,7 +740,6 @@ async def sync_ci_images_cli( data_path=data_path, data_gitref=data_gitref, only_stream=only_stream, - images=images, add_labels=add_labels, assembly=assembly, skip_prs=skip_prs, diff --git a/pyartcd/tests/pipelines/test_build_microshift_bootc.py b/pyartcd/tests/pipelines/test_build_microshift_bootc.py index 3a44f2a2b0..4966b87ebc 100644 --- a/pyartcd/tests/pipelines/test_build_microshift_bootc.py +++ b/pyartcd/tests/pipelines/test_build_microshift_bootc.py @@ -632,56 +632,3 @@ def test_validate_shipment_mr_raises_on_closed_mr(self): pipeline._validate_shipment_mr(shipment_url) self.assertIn("closed", str(ctx.exception)) self.assertIn("not opened", str(ctx.exception)) - - @patch("pyartcd.pipelines.build_microshift_bootc.jenkins.start_build_plashets") - @patch("pyartcd.pipelines.build_microshift_bootc.get_microshift_builds", new_callable=AsyncMock) - @patch("pyartcd.pipelines.build_microshift_bootc.default_release_suffix", return_value="202601290005.p0") - async def test_build_plashet_raises_on_failure(self, mock_release_suffix, mock_get_builds, mock_start_plashets): - """ - Test that _build_plashet_for_bootc raises RuntimeError when the - build-plashets Jenkins job returns a non-SUCCESS result. - """ - # given - mock_start_plashets.return_value = "FAILURE" - pipeline = self._make_pipeline() - pipeline.group_config = {"all_repos": []} - pipeline.force_plashet_sync = True # skip the _rebuild_needed() check - - # Mock plashet config discovery - mock_plashet = Mock() - mock_plashet.disabled = False - mock_plashet.type = "plashet" - mock_plashet.name = "rhel-9-server-microshift-rpms" - with patch("pyartcd.pipelines.build_microshift_bootc.RepoList") as mock_repo_list: - mock_repo_list.model_validate.return_value.root = [mock_plashet] - - # when / then - with self.assertRaises(RuntimeError) as ctx: - await pipeline._build_plashet_for_bootc() - self.assertIn("FAILURE", str(ctx.exception)) - self.assertIn("rhel-9-server-microshift-rpms", str(ctx.exception)) - - @patch("pyartcd.pipelines.build_microshift_bootc.jenkins.start_build_plashets") - @patch("pyartcd.pipelines.build_microshift_bootc.get_microshift_builds", new_callable=AsyncMock) - @patch("pyartcd.pipelines.build_microshift_bootc.default_release_suffix", return_value="202601290005.p0") - async def test_build_plashet_succeeds_on_success(self, mock_release_suffix, mock_get_builds, mock_start_plashets): - """ - Test that _build_plashet_for_bootc completes without error when the - build-plashets Jenkins job returns SUCCESS. - """ - # given - mock_start_plashets.return_value = "SUCCESS" - pipeline = self._make_pipeline() - pipeline.group_config = {"all_repos": []} - pipeline.force_plashet_sync = True # skip the _rebuild_needed() check - - # Mock plashet config discovery - mock_plashet = Mock() - mock_plashet.disabled = False - mock_plashet.type = "plashet" - mock_plashet.name = "rhel-9-server-microshift-rpms" - with patch("pyartcd.pipelines.build_microshift_bootc.RepoList") as mock_repo_list: - mock_repo_list.model_validate.return_value.root = [mock_plashet] - - # when / then - should not raise - await pipeline._build_plashet_for_bootc() diff --git a/pyartcd/tests/pipelines/test_golang_builder_shipment.py b/pyartcd/tests/pipelines/test_golang_builder_shipment.py new file mode 100644 index 0000000000..0b5155ecc7 --- /dev/null +++ b/pyartcd/tests/pipelines/test_golang_builder_shipment.py @@ -0,0 +1,627 @@ +import unittest +from pathlib import Path +from unittest import IsolatedAsyncioTestCase +from unittest.mock import AsyncMock, MagicMock, Mock, patch + +from doozerlib.constants import ART_IMAGES_BASE_APPLICATION +from pyartcd.pipelines.golang_builder_shipment import ( + GOLANG_BUILDER_SHIPMENT_RELEASE_PLAN_MAP, + GolangBuilderShipmentPipeline, + derive_golang_group, + resolve_lifecycle_env, +) + + +class TestResolveReleasePlan(unittest.TestCase): + """Test lifecycle -> releasePlan mapping.""" + + def test_prod_returns_prod_plan(self): + plan = GolangBuilderShipmentPipeline.resolve_release_plan("prod") + self.assertEqual(plan, "ocp-art-golang-builder-prod-rhel9") + + def test_ec_returns_ec_plan(self): + plan = GolangBuilderShipmentPipeline.resolve_release_plan("ec") + self.assertEqual(plan, "ocp-art-golang-builder-ec-rhel9") + + def test_unknown_env_raises(self): + with self.assertRaises(ValueError): + GolangBuilderShipmentPipeline.resolve_release_plan("staging") + + def test_map_keys_are_complete(self): + self.assertIn("prod", GOLANG_BUILDER_SHIPMENT_RELEASE_PLAN_MAP) + self.assertIn("ec", GOLANG_BUILDER_SHIPMENT_RELEASE_PLAN_MAP) + + +class _FakeResponse: + def __init__(self, status, body=""): + self.status = status + self._body = body + + async def text(self): + return self._body + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + pass + + +class _FakeSession: + def __init__(self, response): + self._response = response + + def get(self, *args, **kwargs): + return self._response + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + pass + + +class TestResolveLifecycleEnv(IsolatedAsyncioTestCase): + """Test auto-resolution of prod/ec from ocp-build-data group.yml.""" + + @patch("pyartcd.pipelines.golang_builder_shipment.aiohttp.ClientSession") + async def test_release_phase_returns_prod(self, mock_session_cls): + resp = _FakeResponse(200, "software_lifecycle:\n phase: release\n") + mock_session_cls.return_value = _FakeSession(resp) + result = await resolve_lifecycle_env("4.18") + self.assertEqual(result, "prod") + + @patch("pyartcd.pipelines.golang_builder_shipment.aiohttp.ClientSession") + async def test_pre_release_phase_returns_ec(self, mock_session_cls): + resp = _FakeResponse(200, "software_lifecycle:\n phase: pre-release\n") + mock_session_cls.return_value = _FakeSession(resp) + result = await resolve_lifecycle_env("4.23") + self.assertEqual(result, "ec") + + @patch("pyartcd.pipelines.golang_builder_shipment.aiohttp.ClientSession") + async def test_missing_lifecycle_defaults_to_prod(self, mock_session_cls): + resp = _FakeResponse(200, "vars:\n GOLANG_VERSION: '1.22'\n") + mock_session_cls.return_value = _FakeSession(resp) + result = await resolve_lifecycle_env("4.16") + self.assertEqual(result, "prod") + + @patch("pyartcd.pipelines.golang_builder_shipment.aiohttp.ClientSession") + async def test_http_error_raises(self, mock_session_cls): + resp = _FakeResponse(404) + mock_session_cls.return_value = _FakeSession(resp) + with self.assertRaises(RuntimeError): + await resolve_lifecycle_env("99.99") + + +class TestBasicAuthUrl(unittest.TestCase): + def test_injects_token(self): + url = "https://gitlab.cee.redhat.com/hybrid-platforms/art/ocp-shipment-data.git" + result = GolangBuilderShipmentPipeline.basic_auth_url(url, "mytoken") + self.assertIn("oauth2:mytoken@", result) + self.assertIn("gitlab.cee.redhat.com", result) + self.assertTrue(result.startswith("https://")) + + +class TestPipelineInit(unittest.TestCase): + def _make_runtime(self, dry_run=False): + runtime = Mock() + runtime.dry_run = dry_run + runtime.config = {} + runtime.working_dir = Path("/tmp/test-working") + return runtime + + def test_init_sorts_nvrs(self): + runtime = self._make_runtime() + pipeline = GolangBuilderShipmentPipeline( + runtime=runtime, + ocp_version="4.22", + golang_group="rhel-9-golang-1.25", + nvrs=["golang-builder-container-v1.25-1.el9", "golang-builder-container-v1.25-1.el8"], + ) + self.assertEqual(pipeline.product, "ocp") + self.assertEqual( + pipeline.nvrs, + [ + "golang-builder-container-v1.25-1.el8", + "golang-builder-container-v1.25-1.el9", + ], + ) + # env and release_plan are None until run() resolves them + self.assertIsNone(pipeline.env) + self.assertIsNone(pipeline.release_plan) + + +class TestBuildShipmentConfig(IsolatedAsyncioTestCase): + def _make_pipeline(self, env="prod"): + runtime = Mock() + runtime.dry_run = False + runtime.config = {} + runtime.working_dir = Path("/tmp/test-working") + pipeline = GolangBuilderShipmentPipeline( + runtime=runtime, + ocp_version="4.22", + golang_group="rhel-9-golang-1.25", + nvrs=["golang-builder-container-v1.25-202606220000.el9"], + art_jira="ART-20930", + ) + pipeline.env = env + pipeline.release_plan = pipeline.resolve_release_plan(env) + return pipeline + + @patch("pyartcd.pipelines.golang_builder_shipment.exectools.cmd_gather_async") + @patch("pyartcd.pipelines.golang_builder_shipment.os.unlink") + @patch.dict("os.environ", {}, clear=False) + async def test_build_shipment_config_prod(self, mock_unlink, mock_cmd): + mock_cmd.return_value = ( + 0, + """ +apiVersion: appstudio.redhat.com/v1alpha1 +kind: Snapshot +spec: + application: art-images-base + components: + - name: golang-builder-v1.25-rhel9 + containerImage: quay.io/redhat-user-workloads/ocp-art-tenant/art-images@sha256:abc123 + source: + git: + url: https://github.com/openshift-priv/builder + revision: abc123 +""", + "", + ) + + pipeline = self._make_pipeline(env="prod") + config = await pipeline.build_shipment_config() + + self.assertEqual(config.shipment.metadata.product, "ocp") + self.assertEqual(config.shipment.metadata.application, "art-images-base") + self.assertEqual(config.shipment.metadata.group, "rhel-9-golang-1.25") + self.assertEqual(config.shipment.metadata.assembly, "stream") + self.assertEqual( + config.shipment.environments.prod.releasePlan, + "ocp-art-golang-builder-prod-rhel9", + ) + self.assertEqual(config.shipment.data.releaseNotes.type, "RHBA") + self.assertIn( + "https://redhat.atlassian.net/browse/ART-20930", + config.shipment.data.releaseNotes.references, + ) + + cmd_args = mock_cmd.call_args[0][0] + self.assertTrue(any("--builds-file=" in str(a) for a in cmd_args)) + + @patch("pyartcd.pipelines.golang_builder_shipment.exectools.cmd_gather_async") + @patch("pyartcd.pipelines.golang_builder_shipment.os.unlink") + @patch.dict("os.environ", {}, clear=False) + async def test_build_shipment_config_ec(self, mock_unlink, mock_cmd): + mock_cmd.return_value = ( + 0, + """ +apiVersion: appstudio.redhat.com/v1alpha1 +kind: Snapshot +spec: + application: art-images-base + components: + - name: golang-builder-v1.26-rhel9 + containerImage: quay.io/redhat-user-workloads/ocp-art-tenant/art-images@sha256:def456 + source: + git: + url: https://github.com/openshift-priv/builder + revision: def456 +""", + "", + ) + + pipeline = self._make_pipeline(env="ec") + config = await pipeline.build_shipment_config() + + self.assertEqual( + config.shipment.environments.prod.releasePlan, + "ocp-art-golang-builder-ec-rhel9", + ) + + +class TestCreateShipmentMR(IsolatedAsyncioTestCase): + def _make_pipeline(self, dry_run=False): + runtime = Mock() + runtime.dry_run = dry_run + runtime.config = {} + runtime.working_dir = Path("/tmp/test-working") + pipeline = GolangBuilderShipmentPipeline( + runtime=runtime, + ocp_version="4.22", + golang_group="rhel-9-golang-1.25", + nvrs=["some-nvr"], + art_jira="ART-20930", + ) + pipeline.env = "prod" + pipeline.release_plan = "ocp-art-golang-builder-prod-rhel9" + pipeline._gitlab_token = "test-token" + pipeline.shipment_data_repo = AsyncMock() + pipeline.shipment_data_repo._directory = Path("/tmp/test-working/shipment-data-push") + pipeline.shipment_data_repo.commit_push = AsyncMock(return_value=True) + return pipeline + + def _make_config_mock(self): + config = Mock() + config.shipment.metadata.application = "art-images-base" + config.model_dump.return_value = {"shipment": {}} + return config + + @patch("pyartcd.pipelines.golang_builder_shipment.python_gitlab") + async def test_dry_run_returns_placeholder(self, mock_gitlab): + pipeline = self._make_pipeline(dry_run=True) + pipeline.env = "prod" + config = self._make_config_mock() + + mock_project = MagicMock() + mock_gitlab.Gitlab.return_value.projects.get.return_value = mock_project + + with patch("pathlib.Path.mkdir"): + result = await pipeline.create_shipment_mr(config) + + self.assertIn("placeholder", result) + mock_project.mergerequests.create.assert_not_called() + + @patch("pyartcd.pipelines.golang_builder_shipment.python_gitlab") + async def test_creates_mr_with_correct_title(self, mock_gitlab): + pipeline = self._make_pipeline(dry_run=False) + config = self._make_config_mock() + + mock_project = MagicMock() + mock_mr = MagicMock() + mock_mr.web_url = "https://gitlab.cee.redhat.com/test/-/merge_requests/1" + mock_project.mergerequests.create.return_value = mock_mr + mock_gitlab.Gitlab.return_value.projects.get.return_value = mock_project + + with patch("pathlib.Path.mkdir"): + result = await pipeline.create_shipment_mr(config) + + self.assertEqual(result, mock_mr.web_url) + create_args = mock_project.mergerequests.create.call_args[0][0] + self.assertIn("Golang builder shipment", create_args["title"]) + self.assertEqual(create_args["target_branch"], "main") + + +class TestSetupReposNoToken(IsolatedAsyncioTestCase): + @patch.dict("os.environ", {}, clear=True) + async def test_missing_gitlab_token_raises(self): + runtime = Mock() + runtime.dry_run = False + runtime.config = {} + runtime.working_dir = Path("/tmp/test-working") + pipeline = GolangBuilderShipmentPipeline( + runtime=runtime, + ocp_version="4.22", + golang_group="rhel-9-golang-1.25", + nvrs=["some-nvr"], + ) + with self.assertRaises(ValueError): + await pipeline.setup_repos() + + +class TestSnapshotWithQuayAuth(IsolatedAsyncioTestCase): + @patch("pyartcd.pipelines.golang_builder_shipment.exectools.cmd_gather_async") + @patch("pyartcd.pipelines.golang_builder_shipment.os.unlink") + @patch.dict("os.environ", {"QUAY_AUTH_FILE": "/tmp/quay-auth.json"}) + async def test_pull_secret_flag_added(self, mock_unlink, mock_cmd): + mock_cmd.return_value = ( + 0, + """ +spec: + application: art-images-base + components: + - name: golang-builder-v1.25-rhel9 + containerImage: quay.io/test@sha256:abc + source: + git: + url: https://github.com/openshift-priv/builder + revision: abc123 +""", + "", + ) + runtime = Mock() + runtime.dry_run = False + runtime.config = {} + runtime.working_dir = Path("/tmp/test-working") + pipeline = GolangBuilderShipmentPipeline( + runtime=runtime, + ocp_version="4.22", + golang_group="rhel-9-golang-1.25", + nvrs=["some-nvr"], + ) + await pipeline._create_snapshot() + cmd_args = mock_cmd.call_args[0][0] + self.assertTrue(any("--pull-secret=" in str(a) for a in cmd_args)) + + +class TestApplicationOverride(IsolatedAsyncioTestCase): + """Verify snapshot application is overridden to art-images-base.""" + + @patch("pyartcd.pipelines.golang_builder_shipment.exectools.cmd_gather_async") + @patch("pyartcd.pipelines.golang_builder_shipment.os.unlink") + @patch.dict("os.environ", {}, clear=False) + async def test_elliott_app_overridden_to_art_images_base(self, mock_unlink, mock_cmd): + # elliott returns "rhel-9-golang-1-25" as the application + mock_cmd.return_value = ( + 0, + """ +spec: + application: rhel-9-golang-1-25 + components: + - name: golang-builder-v1.25-rhel9 + containerImage: quay.io/test@sha256:abc + source: + git: + url: https://github.com/openshift-priv/builder + revision: abc123 +""", + "", + ) + runtime = Mock() + runtime.dry_run = False + runtime.config = {} + runtime.working_dir = Path("/tmp/test-working") + pipeline = GolangBuilderShipmentPipeline( + runtime=runtime, + ocp_version="4.22", + golang_group="rhel-9-golang-1.25", + nvrs=["some-nvr"], + art_jira="ART-20930", + ) + pipeline.env = "prod" + pipeline.release_plan = pipeline.resolve_release_plan("prod") + + config = await pipeline.build_shipment_config() + + self.assertEqual(config.shipment.metadata.application, ART_IMAGES_BASE_APPLICATION) + self.assertEqual(config.shipment.snapshot.spec.application, ART_IMAGES_BASE_APPLICATION) + + +class TestCreateSnapshotErrors(IsolatedAsyncioTestCase): + """Test error paths in _create_snapshot.""" + + def _make_pipeline(self): + runtime = Mock() + runtime.dry_run = False + runtime.config = {} + runtime.working_dir = Path("/tmp/test-working") + return GolangBuilderShipmentPipeline( + runtime=runtime, + ocp_version="4.22", + golang_group="rhel-9-golang-1.25", + nvrs=["some-nvr"], + ) + + @patch("pyartcd.pipelines.golang_builder_shipment.exectools.cmd_gather_async") + @patch("pyartcd.pipelines.golang_builder_shipment.os.unlink") + @patch.dict("os.environ", {}, clear=False) + async def test_elliott_nonzero_rc_raises(self, mock_unlink, mock_cmd): + mock_cmd.return_value = (1, "", "elliott error: NVR not found") + pipeline = self._make_pipeline() + with self.assertRaises(RuntimeError) as ctx: + await pipeline._create_snapshot() + self.assertIn("elliott snapshot new failed", str(ctx.exception)) + + @patch("pyartcd.pipelines.golang_builder_shipment.exectools.cmd_gather_async") + @patch("pyartcd.pipelines.golang_builder_shipment.os.unlink") + @patch.dict("os.environ", {}, clear=False) + async def test_empty_stdout_raises(self, mock_unlink, mock_cmd): + mock_cmd.return_value = (0, "", "") + pipeline = self._make_pipeline() + with self.assertRaises(ValueError) as ctx: + await pipeline._create_snapshot() + self.assertIn("invalid output", str(ctx.exception)) + + @patch("pyartcd.pipelines.golang_builder_shipment.exectools.cmd_gather_async") + @patch("pyartcd.pipelines.golang_builder_shipment.os.unlink") + @patch.dict("os.environ", {}, clear=False) + async def test_missing_spec_raises(self, mock_unlink, mock_cmd): + mock_cmd.return_value = (0, "apiVersion: v1\nkind: Snapshot\n", "") + pipeline = self._make_pipeline() + with self.assertRaises(ValueError) as ctx: + await pipeline._create_snapshot() + self.assertIn("missing 'spec'", str(ctx.exception)) + + +class TestCommitPushFailure(IsolatedAsyncioTestCase): + """Test create_shipment_mr when commit_push returns False.""" + + async def test_push_failure_raises(self): + runtime = Mock() + runtime.dry_run = False + runtime.config = {} + runtime.working_dir = Path("/tmp/test-working") + pipeline = GolangBuilderShipmentPipeline( + runtime=runtime, + ocp_version="4.22", + golang_group="rhel-9-golang-1.25", + nvrs=["some-nvr"], + art_jira="ART-20930", + ) + pipeline.env = "prod" + pipeline.release_plan = "ocp-art-golang-builder-prod-rhel9" + pipeline._gitlab_token = "test-token" + pipeline.shipment_data_repo = AsyncMock() + pipeline.shipment_data_repo._directory = Path("/tmp/test-working/shipment-data-push") + pipeline.shipment_data_repo.commit_push = AsyncMock(return_value=False) + + config = Mock() + config.shipment.metadata.application = "art-images-base" + config.model_dump.return_value = {"shipment": {}} + + with patch("pathlib.Path.mkdir"): + with self.assertRaises(RuntimeError) as ctx: + await pipeline.create_shipment_mr(config) + self.assertIn("Failed to push", str(ctx.exception)) + + +class TestRunOrchestration(IsolatedAsyncioTestCase): + """Test run() wiring.""" + + @patch("pyartcd.pipelines.golang_builder_shipment.resolve_lifecycle_env") + async def test_run_calls_steps_in_order(self, mock_resolve_env): + mock_resolve_env.return_value = "prod" + runtime = Mock() + runtime.dry_run = False + runtime.config = {} + runtime.working_dir = Path("/tmp/test-working") + pipeline = GolangBuilderShipmentPipeline( + runtime=runtime, + ocp_version="4.22", + golang_group="rhel-9-golang-1.25", + nvrs=["some-nvr"], + ) + pipeline.setup_working_dir = Mock() + pipeline.setup_repos = AsyncMock() + pipeline.build_shipment_config = AsyncMock(return_value=Mock()) + pipeline.create_shipment_mr = AsyncMock(return_value="https://gitlab.example.com/-/merge_requests/1") + + result = await pipeline.run() + + self.assertEqual(result, "https://gitlab.example.com/-/merge_requests/1") + pipeline.setup_working_dir.assert_called_once() + pipeline.setup_repos.assert_called_once() + pipeline.build_shipment_config.assert_called_once() + pipeline.create_shipment_mr.assert_called_once() + self.assertEqual(pipeline.env, "prod") + self.assertEqual(pipeline.release_plan, "ocp-art-golang-builder-prod-rhel9") + + +class TestResolveLifecycleEnvUnknownPhase(IsolatedAsyncioTestCase): + """Test unknown phase falls back to prod.""" + + @patch("pyartcd.pipelines.golang_builder_shipment.aiohttp.ClientSession") + async def test_unknown_phase_defaults_to_prod(self, mock_session_cls): + resp = _FakeResponse(200, "software_lifecycle:\n phase: maintenance\n") + mock_session_cls.return_value = _FakeSession(resp) + result = await resolve_lifecycle_env("4.14") + self.assertEqual(result, "prod") + + +class TestBuildShipmentConfigNoJira(IsolatedAsyncioTestCase): + """Test build_shipment_config without art_jira.""" + + @patch("pyartcd.pipelines.golang_builder_shipment.exectools.cmd_gather_async") + @patch("pyartcd.pipelines.golang_builder_shipment.os.unlink") + @patch.dict("os.environ", {}, clear=False) + async def test_no_jira_omits_references(self, mock_unlink, mock_cmd): + mock_cmd.return_value = ( + 0, + """ +spec: + application: art-images-base + components: + - name: golang-builder-v1.25-rhel9 + containerImage: quay.io/test@sha256:abc + source: + git: + url: https://github.com/openshift-priv/builder + revision: abc123 +""", + "", + ) + runtime = Mock() + runtime.dry_run = False + runtime.config = {} + runtime.working_dir = Path("/tmp/test-working") + pipeline = GolangBuilderShipmentPipeline( + runtime=runtime, + ocp_version="4.22", + golang_group="rhel-9-golang-1.25", + nvrs=["some-nvr"], + art_jira="", + ) + pipeline.env = "prod" + pipeline.release_plan = pipeline.resolve_release_plan("prod") + + config = await pipeline.build_shipment_config() + + self.assertFalse( + hasattr(config.shipment.data.releaseNotes, 'references') and config.shipment.data.releaseNotes.references + ) + + +class TestSetupReposSuccess(IsolatedAsyncioTestCase): + """Test setup_repos happy path.""" + + @patch.dict("os.environ", {"GITLAB_TOKEN": "test-token"}, clear=False) + async def test_setup_repos_configures_repo(self): + runtime = Mock() + runtime.dry_run = False + runtime.config = {} + runtime.working_dir = Path("/tmp/test-working") + pipeline = GolangBuilderShipmentPipeline( + runtime=runtime, + ocp_version="4.22", + golang_group="rhel-9-golang-1.25", + nvrs=["some-nvr"], + ) + pipeline.shipment_data_repo = AsyncMock() + + await pipeline.setup_repos() + + self.assertEqual(pipeline._gitlab_token, "test-token") + pipeline.shipment_data_repo.setup.assert_called_once() + setup_kwargs = pipeline.shipment_data_repo.setup.call_args[1] + self.assertIn("oauth2:test-token@", setup_kwargs["remote_url"]) + pipeline.shipment_data_repo.fetch_switch_branch.assert_called_once_with("main") + + +class TestDeriveGolangGroup(unittest.TestCase): + """Test golang group derivation from NVR patterns.""" + + def test_from_rpm_nvr(self): + self.assertEqual(derive_golang_group(["golang-1.25.9-1.el9"]), "rhel-9-golang-1.25") + + def test_from_rpm_nvr_el8(self): + self.assertEqual(derive_golang_group(["golang-1.22.3-2.el8"]), "rhel-8-golang-1.22") + + def test_from_konflux_image_nvr(self): + nvr = "openshift-golang-builder-container-v1.25.9-202605121249.p2.gdf787b0.el9" + self.assertEqual(derive_golang_group([nvr]), "rhel-9-golang-1.25") + + def test_unknown_nvr_raises(self): + with self.assertRaises(ValueError): + derive_golang_group(["not-a-golang-nvr-1.0-1.noarch"]) + + def test_init_derives_group_from_image_nvr(self): + runtime = Mock() + runtime.dry_run = False + runtime.config = {} + runtime.working_dir = Path("/tmp/test-working") + pipeline = GolangBuilderShipmentPipeline( + runtime=runtime, + ocp_version="4.22", + nvrs=["openshift-golang-builder-container-v1.25.9-202605121249.p2.gdf787b0.el9"], + ) + self.assertEqual(pipeline.golang_group, "rhel-9-golang-1.25") + + def test_init_derives_group_from_rpm_nvr(self): + runtime = Mock() + runtime.dry_run = False + runtime.config = {} + runtime.working_dir = Path("/tmp/test-working") + pipeline = GolangBuilderShipmentPipeline( + runtime=runtime, + ocp_version="4.22", + nvrs=["golang-1.25.9-1.el9"], + ) + self.assertEqual(pipeline.golang_group, "rhel-9-golang-1.25") + + +class TestShipmentFilePath(unittest.TestCase): + """Verify the shipment YAML file path matches the convention.""" + + def test_path_format(self): + application = "art-images-base" + product = "ocp" + golang_group = "rhel-9-golang-1.25" + env = "prod" + expected_prefix = Path("shipment") / "ocp" / "rhel-9-golang-1.25" / "art-images-base" / "prod" + actual = Path("shipment") / product / golang_group / application / env + self.assertEqual(actual, expected_prefix) + + +if __name__ == "__main__": + unittest.main() diff --git a/pyartcd/tests/pipelines/test_images_health.py b/pyartcd/tests/pipelines/test_images_health.py index a67a3e76cd..9f543458a2 100644 --- a/pyartcd/tests/pipelines/test_images_health.py +++ b/pyartcd/tests/pipelines/test_images_health.py @@ -79,33 +79,17 @@ async def test_with_build_concerns(self): "failing-image", "openshift-4.20", ConcernCode.LATEST_ATTEMPT_FAILED.value, latest_success_idx=3 ), ] - mock_slack_client.say.return_value = {"ts": "thread-123"} + mock_slack_client.say.return_value = {"ts": ""} await pipeline.notify_public_channel() - # Should post a summary message + one thread response per group (2 groups) + # Now only posts a single summary message with dashboard link calls = mock_slack_client.say.call_args_list - self.assertEqual(len(calls), 3) # 1 summary + 2 groups - - # Check main summary message only has alert and dashboard link - summary_args = calls[0][0][0] - self.assertIn(":alert:", summary_args) - self.assertIn("ART Build Failures Dashboard", summary_args) - # Group names should NOT be in the main message - self.assertNotIn("openshift-4.20:", summary_args) - self.assertNotIn("openshift-4.21:", summary_args) - - # Check thread responses contain group summaries - thread_1 = calls[1][0][0] - thread_2 = calls[2][0][0] - - # One should be 4.20, the other 4.21 - self.assertTrue("openshift-4.20" in thread_1 or "openshift-4.20" in thread_2) - self.assertTrue("openshift-4.21" in thread_1 or "openshift-4.21" in thread_2) - self.assertTrue("build failures" in thread_1 or "build failures" in thread_2) - - self.assertEqual(calls[1][1]['thread_ts'], "thread-123") - self.assertEqual(calls[2][1]['thread_ts'], "thread-123") + self.assertEqual(len(calls), 1) + first_call_args = calls[0][0][0] + self.assertIn(":alert:", first_call_args) + self.assertIn("build failures", first_call_args) + self.assertIn("ART Build Failures Dashboard", first_call_args) async def test_with_mixed_failure_types(self): mock_slack_client = self.mock_runtime.new_slack_client.return_value @@ -122,29 +106,19 @@ async def test_with_mixed_failure_types(self): pipeline.rebase_failures = { '4.22': {'rebase-fail-image': {'failure_count': 1, 'jenkins_url': 'http://j/3', 'build_system': 'konflux'}}, } - mock_slack_client.say.return_value = {"ts": "thread-456"} + mock_slack_client.say.return_value = {"ts": ""} await pipeline.notify_public_channel() - # Should post a summary message + one thread response for openshift-4.22 + # Now only posts a single summary message with dashboard link calls = mock_slack_client.say.call_args_list - self.assertEqual(len(calls), 2) # 1 summary + 1 group - - # Check main summary message only has alert and dashboard link + self.assertEqual(len(calls), 1) summary = calls[0][0][0] - self.assertIn(":alert:", summary) + self.assertIn("build failures", summary) + self.assertIn("EC verification failures", summary) + self.assertIn("release to authz failures", summary) + self.assertIn("rebase failures", summary) self.assertIn("ART Build Failures Dashboard", summary) - # Group details should NOT be in the main message - self.assertNotIn("openshift-4.22:", summary) - - # Check thread response contains group summary - thread_message = calls[1][0][0] - self.assertIn("openshift-4.22", thread_message) - self.assertIn("build failures", thread_message) - self.assertIn("EC verification failures", thread_message) - self.assertIn("release to authz failures", thread_message) - self.assertIn("rebase failures", thread_message) - self.assertEqual(calls[1][1]['thread_ts'], "thread-456") async def test_binds_to_configured_channel(self): mock_slack_client = self.mock_runtime.new_slack_client.return_value @@ -199,27 +173,17 @@ async def test_only_ec_failures(self): pipeline.ec_failures = { '4.22': {'image-a': {'failure_count': 5, 'jenkins_url': '', 'pipeline_url': 'http://ec/1'}}, } - mock_slack_client.say.return_value = {"ts": "thread-789"} + mock_slack_client.say.return_value = {"ts": ""} await pipeline.notify_public_channel() - # Should post a summary message + one thread response for openshift-4.22 + # Now only posts a single summary message with dashboard link calls = mock_slack_client.say.call_args_list - self.assertEqual(len(calls), 2) # 1 summary + 1 group - - # Check main summary message only has alert and dashboard link + self.assertEqual(len(calls), 1) summary = calls[0][0][0] - self.assertIn(":alert:", summary) + self.assertIn("EC verification failures", summary) + self.assertNotIn("build failures", summary) self.assertIn("ART Build Failures Dashboard", summary) - # Group details should NOT be in the main message - self.assertNotIn("openshift-4.22:", summary) - - # Check thread response - thread_message = calls[1][0][0] - self.assertIn("openshift-4.22", thread_message) - self.assertIn("EC verification failures", thread_message) - self.assertNotIn("build failures", thread_message) - self.assertEqual(calls[1][1]['thread_ts'], "thread-789") class TestNotifyReleaseChannel(IsolatedAsyncioTestCase): diff --git a/pyartcd/tests/pipelines/test_quay_doomsday_backup.py b/pyartcd/tests/pipelines/test_quay_doomsday_backup.py deleted file mode 100644 index 576527732a..0000000000 --- a/pyartcd/tests/pipelines/test_quay_doomsday_backup.py +++ /dev/null @@ -1,59 +0,0 @@ -"""Unit tests for quay_doomsday_backup pipeline.""" - -from unittest import IsolatedAsyncioTestCase -from unittest.mock import AsyncMock, Mock, patch - -from pyartcd.pipelines.quay_doomsday_backup import QuayDoomsdaySync -from pyartcd.runtime import Runtime -from pyartcd.slack import SlackClient - - -class TestQuayDoomsdaySync(IsolatedAsyncioTestCase): - def setUp(self): - self.runtime = Mock(spec=Runtime) - self.runtime.dry_run = False - self.runtime.logger = Mock() - self.mock_slack_client = Mock(spec=SlackClient) - self.mock_slack_client.say_in_thread = AsyncMock( - return_value={"channel": "C123", "message": {"ts": "1234.5678"}} - ) - self.mock_slack_client._client = Mock() - self.mock_slack_client._client.reactions_add = AsyncMock() - self.runtime.new_slack_client = Mock(return_value=self.mock_slack_client) - self.version = "4.15.5" - self.arches = "x86_64,s390x" - - def _make_pipeline(self) -> QuayDoomsdaySync: - return QuayDoomsdaySync(runtime=self.runtime, version=self.version, arches=self.arches) - - @patch("pyartcd.pipelines.quay_doomsday_backup.mkdirs") - async def test_run_succeeds_when_all_arches_sync(self, _mkdirs_mock): - pipeline = self._make_pipeline() - pipeline.sync_arch = AsyncMock(return_value=True) - - await pipeline.run() - - self.assertEqual(pipeline.sync_arch.await_count, 2) - self.mock_slack_client._client.reactions_add.assert_awaited_once() - - @patch("pyartcd.pipelines.quay_doomsday_backup.mkdirs") - async def test_run_raises_when_some_arches_fail(self, _mkdirs_mock): - pipeline = self._make_pipeline() - pipeline.sync_arch = AsyncMock(side_effect=[True, False]) - - with self.assertRaisesRegex(RuntimeError, "Failed to sync arches for 4.15.5: s390x"): - await pipeline.run() - - self.mock_slack_client.say_in_thread.assert_awaited() - self.mock_slack_client._client.reactions_add.assert_not_awaited() - - @patch("pyartcd.pipelines.quay_doomsday_backup.mkdirs") - async def test_run_raises_when_all_arches_fail(self, _mkdirs_mock): - pipeline = self._make_pipeline() - pipeline.sync_arch = AsyncMock(return_value=False) - - with self.assertRaisesRegex(RuntimeError, "Failed to sync arches for 4.15.5: x86_64, s390x"): - await pipeline.run() - - self.mock_slack_client.say_in_thread.assert_awaited() - self.mock_slack_client._client.reactions_add.assert_not_awaited() diff --git a/pyartcd/tests/pipelines/test_sync_ci_images.py b/pyartcd/tests/pipelines/test_sync_ci_images.py index 3fd5be13b5..d72288565d 100644 --- a/pyartcd/tests/pipelines/test_sync_ci_images.py +++ b/pyartcd/tests/pipelines/test_sync_ci_images.py @@ -124,66 +124,6 @@ def test_validate_skip_prs_already_true_not_changed(self, mock_runtime): assert pipeline.skip_prs is True - def test_validate_auto_sets_skip_prs_for_images(self, mock_runtime): - """Test SKIP_PRS automatically set to true when IMAGES specified.""" - pipeline = SyncCIImagesPipeline( - mock_runtime, - for_release="4.17", - images="ci-openshift-base.rhel10", - skip_prs=False, - ) - assert pipeline.skip_prs is True - - def test_images_parsed_as_list(self, mock_runtime): - """Test comma-separated IMAGES string is parsed into a list.""" - pipeline = SyncCIImagesPipeline( - mock_runtime, - for_release="4.17", - images="ci-openshift-base.rhel10,ci-openshift-base.rhel9", - ) - assert pipeline.images == ["ci-openshift-base.rhel10", "ci-openshift-base.rhel9"] - - def test_images_empty_string_yields_empty_list(self, mock_runtime): - """Test empty IMAGES string yields empty list.""" - pipeline = SyncCIImagesPipeline(mock_runtime, for_release="4.17", images="") - assert pipeline.images == [] - - def test_images_whitespace_handling(self, mock_runtime): - """Test IMAGES strips whitespace around entries.""" - pipeline = SyncCIImagesPipeline( - mock_runtime, - for_release="4.17", - images=" ci-openshift-base.rhel10 , ci-openshift-base.rhel9 ", - ) - assert pipeline.images == ["ci-openshift-base.rhel10", "ci-openshift-base.rhel9"] - - def test_filter_args_with_stream_only(self, mock_runtime): - """Test _filter_args returns only --stream when no images.""" - pipeline = SyncCIImagesPipeline(mock_runtime, for_release="4.17", only_stream="rhel10") - assert pipeline._filter_args == "--stream rhel10" - - def test_filter_args_with_images_only(self, mock_runtime): - """Test _filter_args returns only --image args when no stream.""" - pipeline = SyncCIImagesPipeline(mock_runtime, for_release="4.17", images="ci-openshift-base.rhel10") - assert pipeline._filter_args == "--image ci-openshift-base.rhel10" - - def test_filter_args_with_both(self, mock_runtime): - """Test _filter_args returns both --stream and --image args.""" - pipeline = SyncCIImagesPipeline( - mock_runtime, - for_release="4.17", - only_stream="rhel10", - images="ci-openshift-base.rhel10,ci-openshift-base.rhel9", - ) - assert ( - pipeline._filter_args == "--stream rhel10 --image ci-openshift-base.rhel10 --image ci-openshift-base.rhel9" - ) - - def test_filter_args_empty_when_neither(self, mock_runtime): - """Test _filter_args returns empty string when neither stream nor images.""" - pipeline = SyncCIImagesPipeline(mock_runtime, for_release="4.17") - assert pipeline._filter_args == "" - def test_init_with_custom_data_path(self, mock_runtime): """Test initialization with custom data path and gitref.""" pipeline = SyncCIImagesPipeline( From d5687a3c5f68722c93d0cc15572d2829b1d48002 Mon Sep 17 00:00:00 2001 From: LuLo Date: Tue, 4 Aug 2026 11:10:19 +0200 Subject: [PATCH 2/2] fix(tests): align test expectations with main after rebase Tests for images_health, images_streams image-filtering, and shell_parser arch-conditional parsing were stale after rebase. rh-pre-commit.version: 2.3.2 rh-pre-commit.check-secrets: ENABLED --- doozer/tests/cli/test_images_streams.py | 66 ++++++++++++++++++- .../lockfile_prototype/test_shell_parser.py | 31 ++++++++- pyartcd/tests/pipelines/test_images_health.py | 47 +++++++------ 3 files changed, 123 insertions(+), 21 deletions(-) diff --git a/doozer/tests/cli/test_images_streams.py b/doozer/tests/cli/test_images_streams.py index 079e774318..378cc5c7f9 100644 --- a/doozer/tests/cli/test_images_streams.py +++ b/doozer/tests/cli/test_images_streams.py @@ -278,6 +278,70 @@ def test_get_upstreaming_entries_with_final_user(mock_runtime, mock_image_meta): assert result['ose-test'].final_user == '1001' +# Tests for --image filtering in _get_upstreaming_entries + + +def test_get_upstreaming_entries_image_names_only(mock_runtime, mock_image_meta): + """Test filtering by image_names skips streams and only processes specified images.""" + mock_image_meta.distgit_key = 'ci-openshift-base.rhel10' + mock_image_meta.pull_url.return_value = 'brew-registry.example.com/ci-openshift-base:latest' + mock_runtime.image_map = {'ci-openshift-base.rhel10': mock_image_meta} + + result = images_streams._get_upstreaming_entries(mock_runtime, image_names=['ci-openshift-base.rhel10']) + + assert len(result) == 1 + assert 'ci-openshift-base.rhel10' in result + mock_runtime.ordered_image_metas.assert_not_called() + mock_runtime.get_stream_names.assert_not_called() + + +def test_get_upstreaming_entries_image_not_found(mock_runtime): + """Test error when specified image_name is not in the group.""" + mock_runtime.image_map = {} + + with pytest.raises(IOError, match='Image nonexistent not found in group metadata'): + images_streams._get_upstreaming_entries(mock_runtime, image_names=['nonexistent']) + + +def test_get_upstreaming_entries_image_not_eligible(mocker, mock_runtime): + """Test error when image lacks ci_alignment.upstream_image.""" + ineligible_meta = mocker.MagicMock() + ineligible_meta.config.content.source.ci_alignment.upstream_image = Missing + mock_runtime.image_map = {'ose-cli': ineligible_meta} + + with pytest.raises(IOError, match='not eligible for CI sync'): + images_streams._get_upstreaming_entries(mock_runtime, image_names=['ose-cli']) + + +def test_get_upstreaming_entries_stream_and_image_union(mocker, mock_runtime, mock_image_meta): + """Test that providing both stream_names and image_names returns the union.""" + mock_runtime.streams = { + 'golang': Model({'upstream_image': 'registry.ci.openshift.org/ocp/4.17:golang'}), + } + mock_image_meta.distgit_key = 'ci-openshift-base.rhel10' + mock_image_meta.pull_url.return_value = 'brew-registry.example.com/ci-openshift-base:latest' + mock_runtime.image_map = {'ci-openshift-base.rhel10': mock_image_meta} + + result = images_streams._get_upstreaming_entries( + mock_runtime, stream_names=['golang'], image_names=['ci-openshift-base.rhel10'] + ) + + assert len(result) == 2 + assert 'golang' in result + assert 'ci-openshift-base.rhel10' in result + + +def test_get_upstreaming_entries_image_build_not_found_skips(mocker, mock_runtime, mock_image_meta): + """Test that an image with no build is gracefully skipped.""" + mock_image_meta.distgit_key = 'ci-openshift-base.rhel10' + mock_image_meta.pull_url.side_effect = IOError('No build found') + mock_runtime.image_map = {'ci-openshift-base.rhel10': mock_image_meta} + + result = images_streams._get_upstreaming_entries(mock_runtime, image_names=['ci-openshift-base.rhel10']) + + assert result == {} + + # Tests for images:streams gen-buildconfigs command @@ -295,7 +359,7 @@ def test_gen_buildconfigs_uses_get_upstreaming_entries(): source = inspect.getsource(callback) # Verify it calls _get_upstreaming_entries - assert '_get_upstreaming_entries(runtime, streams)' in source + assert '_get_upstreaming_entries(runtime, streams' in source # Verify it uses the configuration fields from entries assert 'config.transform' in source diff --git a/doozer/tests/lockfile_prototype/test_shell_parser.py b/doozer/tests/lockfile_prototype/test_shell_parser.py index 190b5a2b4f..55fd954d10 100644 --- a/doozer/tests/lockfile_prototype/test_shell_parser.py +++ b/doozer/tests/lockfile_prototype/test_shell_parser.py @@ -153,12 +153,41 @@ def test_subshell_arch_conditional_var(self): "yum -y install pciutils hwdata kmod $ARCH_DEP_PKGS" ] common, arch = extract_packages_from_run_commands(run_values) - self.assertIn("mstflint", common) + self.assertNotIn("mstflint", common) self.assertIn("pciutils", common) self.assertIn("hwdata", common) self.assertIn("kmod", common) + for a in ("x86_64", "aarch64", "ppc64le"): + self.assertIn("mstflint", arch.get(a, [])) + self.assertNotIn("s390x", arch) + + def test_subshell_arch_conditional_var_eq(self): + run_values = [ + 'SPECIAL=$(if [ "$(uname -m)" == "x86_64" ]; then echo -n intel-pkg ; fi) && ' + "yum -y install base-pkg $SPECIAL" + ] + common, arch = extract_packages_from_run_commands(run_values) + self.assertNotIn("intel-pkg", common) + self.assertIn("base-pkg", common) + self.assertEqual(arch, {"x86_64": ["intel-pkg"]}) + + def test_subshell_no_arch_condition(self): + run_values = ['EXTRA=$(echo extra-pkg) && yum -y install base-pkg $EXTRA'] + common, arch = extract_packages_from_run_commands(run_values) + self.assertIn("extra-pkg", common) + self.assertIn("base-pkg", common) self.assertEqual(arch, {}) + def test_subshell_arch_conditional_var_go_arch(self): + run_values = [ + 'SPECIAL=$(if [ "$(go env GOARCH)" == "amd64" ]; then echo -n x86-only ; fi) && ' + "yum -y install common-pkg $SPECIAL" + ] + common, arch = extract_packages_from_run_commands(run_values) + self.assertNotIn("x86-only", common) + self.assertIn("common-pkg", common) + self.assertEqual(arch, {"x86_64": ["x86-only"]}) + def test_if_else_var_with_nested_resolution(self): run_values = [ 'if yum install -y iotop >/dev/null 2>&1; then IOTOP_PKG="iotop"; ' diff --git a/pyartcd/tests/pipelines/test_images_health.py b/pyartcd/tests/pipelines/test_images_health.py index 9f543458a2..e345b8b00a 100644 --- a/pyartcd/tests/pipelines/test_images_health.py +++ b/pyartcd/tests/pipelines/test_images_health.py @@ -79,17 +79,13 @@ async def test_with_build_concerns(self): "failing-image", "openshift-4.20", ConcernCode.LATEST_ATTEMPT_FAILED.value, latest_success_idx=3 ), ] - mock_slack_client.say.return_value = {"ts": ""} + mock_slack_client.say.return_value = {"ts": "thread-123"} await pipeline.notify_public_channel() - # Now only posts a single summary message with dashboard link + # Should post a summary message + one thread response per group (2 groups) calls = mock_slack_client.say.call_args_list - self.assertEqual(len(calls), 1) - first_call_args = calls[0][0][0] - self.assertIn(":alert:", first_call_args) - self.assertIn("build failures", first_call_args) - self.assertIn("ART Build Failures Dashboard", first_call_args) + self.assertEqual(len(calls), 3) # 1 summary + 2 groups async def test_with_mixed_failure_types(self): mock_slack_client = self.mock_runtime.new_slack_client.return_value @@ -106,20 +102,26 @@ async def test_with_mixed_failure_types(self): pipeline.rebase_failures = { '4.22': {'rebase-fail-image': {'failure_count': 1, 'jenkins_url': 'http://j/3', 'build_system': 'konflux'}}, } - mock_slack_client.say.return_value = {"ts": ""} + mock_slack_client.say.return_value = {"ts": "thread-456"} await pipeline.notify_public_channel() - # Now only posts a single summary message with dashboard link + # Should post a summary message + one thread response for openshift-4.22 calls = mock_slack_client.say.call_args_list - self.assertEqual(len(calls), 1) + self.assertEqual(len(calls), 2) # 1 summary + 1 group + summary = calls[0][0][0] - self.assertIn("build failures", summary) - self.assertIn("EC verification failures", summary) - self.assertIn("release to authz failures", summary) - self.assertIn("rebase failures", summary) + self.assertIn(":alert:", summary) self.assertIn("ART Build Failures Dashboard", summary) + thread_message = calls[1][0][0] + self.assertIn("openshift-4.22", thread_message) + self.assertIn("build failures", thread_message) + self.assertIn("EC verification failures", thread_message) + self.assertIn("release to authz failures", thread_message) + self.assertIn("rebase failures", thread_message) + self.assertEqual(calls[1][1]['thread_ts'], "thread-456") + async def test_binds_to_configured_channel(self): mock_slack_client = self.mock_runtime.new_slack_client.return_value pipeline = _make_pipeline(self.mock_runtime, public_channel="#my-custom-channel") @@ -173,17 +175,24 @@ async def test_only_ec_failures(self): pipeline.ec_failures = { '4.22': {'image-a': {'failure_count': 5, 'jenkins_url': '', 'pipeline_url': 'http://ec/1'}}, } - mock_slack_client.say.return_value = {"ts": ""} + mock_slack_client.say.return_value = {"ts": "thread-789"} await pipeline.notify_public_channel() - # Now only posts a single summary message with dashboard link + # Should post a summary message + one thread response for openshift-4.22 calls = mock_slack_client.say.call_args_list - self.assertEqual(len(calls), 1) + self.assertEqual(len(calls), 2) # 1 summary + 1 group + summary = calls[0][0][0] - self.assertIn("EC verification failures", summary) - self.assertNotIn("build failures", summary) + self.assertIn(":alert:", summary) self.assertIn("ART Build Failures Dashboard", summary) + self.assertNotIn("openshift-4.22:", summary) + + thread_message = calls[1][0][0] + self.assertIn("openshift-4.22", thread_message) + self.assertIn("EC verification failures", thread_message) + self.assertNotIn("build failures", thread_message) + self.assertEqual(calls[1][1]['thread_ts'], "thread-789") class TestNotifyReleaseChannel(IsolatedAsyncioTestCase):