From 5a44b0a4f26eafdc62a7d11ac30f3b077e0d9298 Mon Sep 17 00:00:00 2001 From: "D. Paolella" Date: Tue, 30 Jun 2026 11:03:21 +0200 Subject: [PATCH] Add support for ignorable repos in scan-sources Implements the ability to mark repos as ignorable to prevent mass rebuilds from RHEL base package updates. rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED --- artcommon/artcommonlib/config/repo.py | 5 +++ doozer/doozerlib/build_info.py | 41 +++++++++++++++++++ doozer/doozerlib/repos.py | 4 ++ doozer/doozerlib/rhcos.py | 22 ++++++++-- doozer/doozerlib/runtime.py | 9 +++- doozer/tests/test_rhcos_optimized.py | 2 + .../tests/pipelines/test_ocp4_scan_konflux.py | 2 + 7 files changed, 80 insertions(+), 5 deletions(-) diff --git a/artcommon/artcommonlib/config/repo.py b/artcommon/artcommonlib/config/repo.py index 001aac6ab1..d1b057756f 100644 --- a/artcommon/artcommonlib/config/repo.py +++ b/artcommon/artcommonlib/config/repo.py @@ -51,6 +51,10 @@ class RepoSync(BaseModel): latest_only: bool = True +class ScanSources(BaseModel): + ignorable: bool = False + + class Repo(BaseModel): name: str disabled: bool = False @@ -59,6 +63,7 @@ class Repo(BaseModel): conf: dict | None = None content_set: ContentSet | None = None reposync: RepoSync = RepoSync() + scan_sources: ScanSources | None = None def construct_download_url( self, diff --git a/doozer/doozerlib/build_info.py b/doozer/doozerlib/build_info.py index e64e0f96b2..8dc16ed671 100644 --- a/doozer/doozerlib/build_info.py +++ b/doozer/doozerlib/build_info.py @@ -149,6 +149,27 @@ async def find_non_latest_rpms(self, rpms_to_check: Optional[List[Dict]] = None) meta.distgit_key, ) return [] + + # Filter out ignorable repos (ART-14091) + # Ignorable repos (e.g., baseos, appstream) don't trigger rebuilds to avoid mass rebuilds + non_ignorable_repos = [] + for repo_name in enabled_repos: + repo = group_repos[repo_name] + # Check if repo has scan_sources.ignorable set to true + if repo._data.get('scan_sources', {}).get('ignorable', False): + logger.info(f'Ignoring repo {repo_name} for RPM change detection (marked as ignorable)') + else: + non_ignorable_repos.append(repo_name) + + if not non_ignorable_repos: + logger.info( + "All enabled repos for %s are marked as ignorable; skipping RPM change detection", + meta.distgit_key, + ) + return [] + + enabled_repos = non_ignorable_repos + logger.info( "Fetching repodatas for enabled repos %s", ", ".join(f"{repo_name}-{arch}" for repo_name in enabled_repos) ) @@ -752,6 +773,26 @@ async def find_non_latest_rpms(self, package_rpm_finder: PackageRpmFinder): ) return {} + # Filter out ignorable repos (ART-14091) + # Ignorable repos (e.g., baseos, appstream) don't trigger rebuilds to avoid mass rebuilds + non_ignorable_repos = [] + for repo_name in enabled_repos: + repo = group_repos[repo_name] + # Check if repo has scan_sources.ignorable set to true + if repo._data.get('scan_sources', {}).get('ignorable', False): + logger.info(f'Ignoring repo {repo_name} for RPM change detection (marked as ignorable)') + else: + non_ignorable_repos.append(repo_name) + + if not non_ignorable_repos: + logger.info( + "All enabled repos for %s are marked as ignorable; skipping RPM change detection", + meta.distgit_key, + ) + return {} + + enabled_repos = non_ignorable_repos + for arch in self._build_record.arches: repodatas = await asyncio.gather( *(group_repos[repo_name].get_repodata(arch) for repo_name in enabled_repos) diff --git a/doozer/doozerlib/repos.py b/doozer/doozerlib/repos.py index 20c53f8db9..51b9293b96 100644 --- a/doozer/doozerlib/repos.py +++ b/doozer/doozerlib/repos.py @@ -96,6 +96,10 @@ def from_repo_config( if repo_config.reposync: repo_dict['reposync'] = repo_config.reposync.model_dump(exclude_none=True) + # Add scan_sources if present (ART-14091) + if repo_config.scan_sources: + repo_dict['scan_sources'] = repo_config.scan_sources.model_dump(exclude_none=True) + return Repo(repo_config.name, repo_dict, list(arches), gpgcheck) def __init__(self, name: str, data: Dict, valid_arches: List[str], gpgcheck: bool = True): diff --git a/doozer/doozerlib/rhcos.py b/doozer/doozerlib/rhcos.py index 83e6b907d8..2c1c13958b 100644 --- a/doozer/doozerlib/rhcos.py +++ b/doozer/doozerlib/rhcos.py @@ -548,12 +548,28 @@ async def find_non_latest_rpms(self, exclude_rhel: Optional[bool] = False) -> Li raise ValueError("RHCOS build repos need to be defined in group config rhcos.enabled_repos.") enabled_repos = enabled_repos.primitive() - enabled_repos_rhel10 = [repo for repo in enabled_repos if "rhel-10" in repo] - enabled_repos_rhel9 = [repo for repo in enabled_repos if "rhel-10" not in repo] - group_repos = self.runtime.repos arch = self.brew_arch + # Filter out ignorable repos (ART-14091) + # Ignorable repos (e.g., baseos, appstream) don't trigger rebuilds to avoid mass rebuilds + non_ignorable_repos = [] + for repo_name in enabled_repos: + repo = group_repos[repo_name] + if repo._data.get('scan_sources', {}).get('ignorable', False): + logger.info(f'Ignoring repo {repo_name} for RHCOS RPM change detection (marked as ignorable)') + else: + non_ignorable_repos.append(repo_name) + + if not non_ignorable_repos: + logger.warning("All RHCOS enabled repos are marked as ignorable; skipping RPM change detection") + return [] + + enabled_repos = non_ignorable_repos + + enabled_repos_rhel10 = [repo for repo in enabled_repos if "rhel-10" in repo] + enabled_repos_rhel9 = [repo for repo in enabled_repos if "rhel-10" not in repo] + logger.info( "Fetching repodatas for enabled repos %s", ", ".join(f"{repo_name}-{arch}" for repo_name in enabled_repos) ) diff --git a/doozer/doozerlib/runtime.py b/doozer/doozerlib/runtime.py index 861b7ed436..492ef2ad22 100644 --- a/doozer/doozerlib/runtime.py +++ b/doozer/doozerlib/runtime.py @@ -25,7 +25,7 @@ ) from artcommonlib.config import BuildDataLoader from artcommonlib.config.plashet import PlashetConfig -from artcommonlib.config.repo import ContentSet, Repo, RepoList, RepoSync +from artcommonlib.config.repo import ContentSet, Repo, RepoList, RepoSync, ScanSources from artcommonlib.model import Missing, Model from artcommonlib.pushd import Dir from artcommonlib.runtime import GroupRuntime @@ -356,7 +356,7 @@ def _get_repos_config(self) -> RepoList: new_repos = [] for repo_name, repo_data in old_repos.items(): - # Parse content_set and reposync if present + # Parse content_set, reposync, and scan_sources if present content_set = None if 'content_set' in repo_data: content_set = ContentSet.model_validate(repo_data['content_set']) @@ -365,6 +365,10 @@ def _get_repos_config(self) -> RepoList: if 'reposync' in repo_data: reposync = RepoSync.model_validate(repo_data['reposync']) + scan_sources = None + if 'scan_sources' in repo_data: + scan_sources = ScanSources.model_validate(repo_data['scan_sources']) + # Create Repo object using constructor repo = Repo( name=repo_name, @@ -373,6 +377,7 @@ def _get_repos_config(self) -> RepoList: conf=repo_data.get('conf'), content_set=content_set, reposync=reposync, + scan_sources=scan_sources, ) new_repos.append(repo) diff --git a/doozer/tests/test_rhcos_optimized.py b/doozer/tests/test_rhcos_optimized.py index eea2cfe3f5..5e981164fe 100644 --- a/doozer/tests/test_rhcos_optimized.py +++ b/doozer/tests/test_rhcos_optimized.py @@ -32,7 +32,9 @@ async def _run_test(): # Mock repos repo9 = MagicMock() + repo9._data.get.return_value = {} # scan_sources not set, so not ignorable repo10 = MagicMock() + repo10._data.get.return_value = {} # scan_sources not set, so not ignorable self.runtime.repos = {"rhel-9-baseos": repo9, "rhel-10-baseos": repo10} # Mock repodata diff --git a/pyartcd/tests/pipelines/test_ocp4_scan_konflux.py b/pyartcd/tests/pipelines/test_ocp4_scan_konflux.py index 2ce122fe6e..faf80d0d3c 100644 --- a/pyartcd/tests/pipelines/test_ocp4_scan_konflux.py +++ b/pyartcd/tests/pipelines/test_ocp4_scan_konflux.py @@ -25,6 +25,7 @@ def _make_pipeline(self): assembly='stream', data_gitref='', image_list='', + skip_rpms=False, ) @patch.dict(os.environ, {'KUBECONFIG': '/path/to/kubeconfig'}) @@ -150,6 +151,7 @@ def setUp(self): assembly="stream", data_gitref="", image_list="", + skip_rpms=False, ) async def test_run_invokes_bridge_bug_mirroring(self):