Skip to content

ART-14091: Add support for ignorable repos in scan-sources #3116

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
locriandev wants to merge 1 commit into
base: main
Choose a base branch
from
Open

ART-14091: Add support for ignorable repos in scan-sources #3116

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions artcommon/artcommonlib/config/repo.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,10 @@ class RepoSync(BaseModel):
latest_only: bool = True


class ScanSources(BaseModel):
ignorable: bool = False


class Repo(BaseModel):
name: str
disabled: bool = False
Expand All @@ -59,6 +63,7 @@ class Repo(BaseModel):
conf: dict | None = None
content_set: ContentSet | None = None
reposync: RepoSync = RepoSync()
scan_sources: ScanSources | None = None

def construct_download_url(
self,
Expand Down
41 changes: 41 additions & 0 deletions doozer/doozerlib/build_info.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,27 @@ async def find_non_latest_rpms(self, rpms_to_check: Optional[List[Dict]] = None)
meta.distgit_key,
)
return []

# Filter out ignorable repos (ART-14091)
# Ignorable repos (e.g., baseos, appstream) don't trigger rebuilds to avoid mass rebuilds
non_ignorable_repos = []
for repo_name in enabled_repos:
repo = group_repos[repo_name]
# Check if repo has scan_sources.ignorable set to true
if repo._data.get('scan_sources', {}).get('ignorable', False):
logger.info(f'Ignoring repo {repo_name} for RPM change detection (marked as ignorable)')
else:
non_ignorable_repos.append(repo_name)

if not non_ignorable_repos:
logger.info(
"All enabled repos for %s are marked as ignorable; skipping RPM change detection",
meta.distgit_key,
)
return []

enabled_repos = non_ignorable_repos

logger.info(
"Fetching repodatas for enabled repos %s", ", ".join(f"{repo_name}-{arch}" for repo_name in enabled_repos)
)
Expand Down Expand Up @@ -752,6 +773,26 @@ async def find_non_latest_rpms(self, package_rpm_finder: PackageRpmFinder):
)
return {}

# Filter out ignorable repos (ART-14091)
# Ignorable repos (e.g., baseos, appstream) don't trigger rebuilds to avoid mass rebuilds
non_ignorable_repos = []
for repo_name in enabled_repos:
repo = group_repos[repo_name]
# Check if repo has scan_sources.ignorable set to true
if repo._data.get('scan_sources', {}).get('ignorable', False):
logger.info(f'Ignoring repo {repo_name} for RPM change detection (marked as ignorable)')
else:
non_ignorable_repos.append(repo_name)

if not non_ignorable_repos:
logger.info(
"All enabled repos for %s are marked as ignorable; skipping RPM change detection",
meta.distgit_key,
)
return {}

enabled_repos = non_ignorable_repos

for arch in self._build_record.arches:
repodatas = await asyncio.gather(
*(group_repos[repo_name].get_repodata(arch) for repo_name in enabled_repos)
Expand Down
4 changes: 4 additions & 0 deletions doozer/doozerlib/repos.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,10 @@ def from_repo_config(
if repo_config.reposync:
repo_dict['reposync'] = repo_config.reposync.model_dump(exclude_none=True)

# Add scan_sources if present (ART-14091)
if repo_config.scan_sources:
repo_dict['scan_sources'] = repo_config.scan_sources.model_dump(exclude_none=True)

return Repo(repo_config.name, repo_dict, list(arches), gpgcheck)

def __init__(self, name: str, data: Dict, valid_arches: List[str], gpgcheck: bool = True):
Expand Down
22 changes: 19 additions & 3 deletions doozer/doozerlib/rhcos.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -548,12 +548,28 @@ async def find_non_latest_rpms(self, exclude_rhel: Optional[bool] = False) -> Li
raise ValueError("RHCOS build repos need to be defined in group config rhcos.enabled_repos.")
enabled_repos = enabled_repos.primitive()

enabled_repos_rhel10 = [repo for repo in enabled_repos if "rhel-10" in repo]
enabled_repos_rhel9 = [repo for repo in enabled_repos if "rhel-10" not in repo]

group_repos = self.runtime.repos
arch = self.brew_arch

# Filter out ignorable repos (ART-14091)
# Ignorable repos (e.g., baseos, appstream) don't trigger rebuilds to avoid mass rebuilds
non_ignorable_repos = []
for repo_name in enabled_repos:
repo = group_repos[repo_name]
if repo._data.get('scan_sources', {}).get('ignorable', False):
logger.info(f'Ignoring repo {repo_name} for RHCOS RPM change detection (marked as ignorable)')
else:
non_ignorable_repos.append(repo_name)

if not non_ignorable_repos:
logger.warning("All RHCOS enabled repos are marked as ignorable; skipping RPM change detection")
return []

enabled_repos = non_ignorable_repos

enabled_repos_rhel10 = [repo for repo in enabled_repos if "rhel-10" in repo]
enabled_repos_rhel9 = [repo for repo in enabled_repos if "rhel-10" not in repo]

logger.info(
"Fetching repodatas for enabled repos %s", ", ".join(f"{repo_name}-{arch}" for repo_name in enabled_repos)
)
Expand Down
9 changes: 7 additions & 2 deletions doozer/doozerlib/runtime.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
)
from artcommonlib.config import BuildDataLoader
from artcommonlib.config.plashet import PlashetConfig
from artcommonlib.config.repo import ContentSet, Repo, RepoList, RepoSync
from artcommonlib.config.repo import ContentSet, Repo, RepoList, RepoSync, ScanSources
from artcommonlib.model import Missing, Model
from artcommonlib.pushd import Dir
from artcommonlib.runtime import GroupRuntime
Expand Down Expand Up @@ -356,7 +356,7 @@ def _get_repos_config(self) -> RepoList:
new_repos = []

for repo_name, repo_data in old_repos.items():
# Parse content_set and reposync if present
# Parse content_set, reposync, and scan_sources if present
content_set = None
if 'content_set' in repo_data:
content_set = ContentSet.model_validate(repo_data['content_set'])
Expand All @@ -365,6 +365,10 @@ def _get_repos_config(self) -> RepoList:
if 'reposync' in repo_data:
reposync = RepoSync.model_validate(repo_data['reposync'])

scan_sources = None
if 'scan_sources' in repo_data:
scan_sources = ScanSources.model_validate(repo_data['scan_sources'])
Comment on lines +368 to +370

@coderabbitai coderabbitai Bot Jun 30, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🗄️ Data Integrity & Integration | 🟡 Minor | ⚡ Quick win

Preserve null for optional scan_sources.

Line 369 validates whenever the key exists, so old-style group.yml now rejects scan_sources: null even though Repo.scan_sources is optional. Treat None the same as an omitted field before calling model_validate.

Suggested fix 
-                scan_sources = None
-                if 'scan_sources' in repo_data:
-                    scan_sources = ScanSources.model_validate(repo_data['scan_sources'])
+                scan_sources = None
+                scan_sources_data = repo_data.get('scan_sources')
+                if scan_sources_data not in (None, Missing):
+                    scan_sources = ScanSources.model_validate(scan_sources_data)
🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@doozer/doozerlib/runtime.py` around lines 368 - 370, The optional
scan_sources handling in runtime.py is too strict because Repo.scan_sources can
be null, but the current scan_sources setup always calls
ScanSources.model_validate whenever the key exists. Update the repo_data parsing
around scan_sources in the runtime logic so that a present-but-None value is
treated the same as a missing field and skipped before validation, while still
validating only non-null values through ScanSources.model_validate.


# Create Repo object using constructor
repo = Repo(
name=repo_name,
Expand All @@ -373,6 +377,7 @@ def _get_repos_config(self) -> RepoList:
conf=repo_data.get('conf'),
content_set=content_set,
reposync=reposync,
scan_sources=scan_sources,
)
new_repos.append(repo)

Expand Down
2 changes: 2 additions & 0 deletions doozer/tests/test_rhcos_optimized.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,9 @@ async def _run_test():

# Mock repos
repo9 = MagicMock()
repo9._data.get.return_value = {} # scan_sources not set, so not ignorable
repo10 = MagicMock()
repo10._data.get.return_value = {} # scan_sources not set, so not ignorable
self.runtime.repos = {"rhel-9-baseos": repo9, "rhel-10-baseos": repo10}

# Mock repodata
Expand Down
2 changes: 2 additions & 0 deletions pyartcd/tests/pipelines/test_ocp4_scan_konflux.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ def _make_pipeline(self):
assembly='stream',
data_gitref='',
image_list='',
skip_rpms=False,
)

@patch.dict(os.environ, {'KUBECONFIG': '/path/to/kubeconfig'})
Expand Down Expand Up @@ -150,6 +151,7 @@ def setUp(self):
assembly="stream",
data_gitref="",
image_list="",
skip_rpms=False,
)

async def test_run_invokes_bridge_bug_mirroring(self):
Expand Down
Loading