Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 21 additions & 3 deletions ocp-build-data-validator/tests/test_schema/test_group_schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,6 @@ def test_validate_with_mismatched_bridge_release_basis_group(self):
self.assertIn("must be 'openshift-5.0'", group_schema.validate("group.yml", invalid_data))

def test_validate_with_templated_group_name_and_valid_bridge_release(self):
# Real group.yml files always use an unresolved "{MAJOR}.{MINOR}" template for
# `name`, substituted at runtime by doozer/elliott. The validator must resolve
# it using `vars` before comparing against `bridge_release.basis_group`.
valid_data = {
"name": "openshift-{MAJOR}.{MINOR}",
"vars": {"MAJOR": 4, "MINOR": 23},
Expand All @@ -84,3 +81,24 @@ def test_validate_with_templated_group_name_and_mismatched_bridge_release(self):
},
}
self.assertIn("must be 'openshift-5.0'", group_schema.validate("group.yml", invalid_data))

def test_validate_with_okd_enabled_flag(self):
valid_data = {
"name": "openshift-4.21",
"vars": {"MAJOR": 4, "MINOR": 21},
"okd": {
"enabled": True,
"konflux": {"build_priority": 8},
},
}
self.assertEqual("", group_schema.validate("group.yml", valid_data))

def test_validate_with_invalid_okd_enabled_flag(self):
invalid_data = {
"name": "openshift-4.21",
"vars": {"MAJOR": 4, "MINOR": 21},
"okd": {
"enabled": "yes",
},
}
self.assertIn("'yes' is not of type 'boolean'", group_schema.validate("group.yml", invalid_data))
Original file line number Diff line number Diff line change
Expand Up @@ -647,6 +647,16 @@
"$ref": "#/properties/okd/properties/branch"
},
"branch-": {},
"enabled": {
"type": "boolean"
},
"enabled!": {
"$ref": "#/properties/okd/properties/enabled"
},
"enabled?": {
"$ref": "#/properties/okd/properties/enabled"
},
"enabled-": {},
"konflux": {
"type": "object",
"properties": {
Expand Down
7 changes: 0 additions & 7 deletions pyartcd/pyartcd/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,10 +41,3 @@
KONFLUX_IMAGE_BUILD_PLR_TEMPLATE_URL_FORMAT = "https://api.github.com/repos/{owner}/art-konflux-template/contents/.tekton/art-konflux-template-push.yaml?ref={branch_name}" # Konflux PipelineRun (PLR) template for image builds
KONFLUX_BUNDLE_BUILD_PLR_TEMPLATE_URL_FORMAT = "https://api.github.com/repos/{owner}/art-konflux-template/contents/.tekton/art-bundle-konflux-template-push.yaml?ref={branch_name}" # Konflux PipelineRun (PLR) template for bundle builds
KONFLUX_FBC_BUILD_PLR_TEMPLATE_URL_FORMAT = "https://api.github.com/repos/{owner}/art-konflux-template/contents/.tekton/art-fbc-konflux-template-push.yaml?ref={branch_name}" # Konflux PipelineRun (PLR) template for FBC builds

# OKD build triggering is enabled only for these OCP versions
OKD_ENABLED_VERSIONS = [
'4.21',
'4.22',
'5.0',
]
44 changes: 42 additions & 2 deletions pyartcd/pyartcd/pipelines/okd_images_health.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,13 @@

import click
from artcommonlib import exectools
from artcommonlib.constants import ACTIVE_OCP_VERSIONS
from doozerlib.cli.images_health import DELTA_DAYS, LIMIT_BUILD_RESULTS, ConcernCode
from doozerlib.constants import ART_BUILD_FAILURES_URL, ART_BUILD_HISTORY_URL

from pyartcd import util
from pyartcd.cli import cli, click_coroutine, pass_runtime
from pyartcd.constants import OCP_BUILD_DATA_URL, OKD_ENABLED_VERSIONS
from pyartcd.constants import OCP_BUILD_DATA_URL
from pyartcd.runtime import Runtime

OKD_GROUP_TEMPLATE = "okd-{}"
Expand All @@ -30,7 +31,7 @@ def __init__(
assembly: str,
):
self.runtime = runtime
self.versions = versions.split(',') if versions else OKD_ENABLED_VERSIONS
self._versions_param = versions
self.doozer_working = self.runtime.working_dir / "doozer_working"
self.send_to_release_channel = send_to_release_channel
self.send_to_okd_channel = send_to_okd_channel
Expand All @@ -44,7 +45,46 @@ def __init__(
self.scanned_versions = []
self.rebase_failures = {} # version -> {image: {failure_count, url}}

def _doozer_base_command(self, version: str) -> list[str]:
group_param = f'openshift-{version}'
if self.data_gitref:
group_param += f'@{self.data_gitref}'
return [
'doozer',
f'--working-dir={self.doozer_working}-{version}',
f'--data-path={self.data_path}',
f'--group={group_param}',
f'--assembly={self.assembly}',
'--build-system=konflux',
'--variant=okd',
]

async def _resolve_versions(self) -> list[str]:
if self._versions_param:
candidates = [v.strip() for v in self._versions_param.split(',') if v.strip()]
else:
candidates = list(ACTIVE_OCP_VERSIONS)
self.runtime.logger.info(
'No --versions provided; probing ACTIVE_OCP_VERSIONS for okd.enabled in build-data'
)

enabled_versions = []
for version in candidates:
if await util.is_okd_version_enabled(self._doozer_base_command(version)):
enabled_versions.append(version)
else:
self.runtime.logger.info(
'Version %s is not enabled for OKD (set okd.enabled: true in group.yml on openshift-%s). Skipping.',
version,
version,
)
return enabled_versions

async def run(self):
self.versions = await self._resolve_versions()
if not self.versions:
self.runtime.logger.info('No OKD-enabled versions to monitor; skipping health report')
return
await asyncio.gather(*(self.get_report(v) for v in self.versions))
await asyncio.gather(*(self.get_rebase_failures(v) for v in self.versions))
self.runtime.logger.info('Found %s concerns', len(self.report))
Expand Down
14 changes: 7 additions & 7 deletions pyartcd/pyartcd/pipelines/okd_scan.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
import yaml
from artcommonlib import exectools

from pyartcd import constants, jenkins, locks
from pyartcd import constants, jenkins, locks, util
from pyartcd.cli import cli, click_coroutine, pass_runtime
from pyartcd.locks import Lock
from pyartcd.runtime import Runtime
Expand Down Expand Up @@ -88,12 +88,14 @@ async def run(self):
# If we get here, lock could be acquired
self.skipped = False

# Early exit if version not enabled for OKD
if self.version not in constants.OKD_ENABLED_VERSIONS:
self._check_params()

# Early exit if version not enabled for OKD in build-data
if not await util.is_okd_version_enabled(self.doozer_base_command):
self.logger.info(
'Version %s is not enabled for OKD (enabled versions: %s). Skipping scan.',
'Version %s is not enabled for OKD (set okd.enabled: true in group.yml on openshift-%s). Skipping scan.',
self.version,
self.version,
constants.OKD_ENABLED_VERSIONS,
)
return

Expand All @@ -103,8 +105,6 @@ async def run(self):
scan_info += f'@{self.data_gitref}'
self.logger.info(scan_info)

self._check_params()

# Scan for changes
await self._scan_sources()

Expand Down
60 changes: 58 additions & 2 deletions pyartcd/pyartcd/pipelines/scheduled/schedule_okd_scan.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,22 @@

from pyartcd import jenkins, util
from pyartcd.cli import cli, click_coroutine, pass_runtime
from pyartcd.constants import OCP_BUILD_DATA_URL
from pyartcd.locks import Lock, LockManager
from pyartcd.runtime import Runtime


async def _is_version_okd_enabled(runtime: Runtime, version: str) -> bool:
doozer_cmd = util.okd_doozer_base_command(
version=version,
assembly='stream',
data_path=OCP_BUILD_DATA_URL,
data_gitref='',
working_dir=runtime.working_dir / f'doozer_working-{version}',
)
return await util.is_okd_version_enabled(doozer_cmd)


async def run_for(version: str, runtime: Runtime, lock_manager: LockManager):
"""
Schedule OKD scan for a specific version if conditions are met.
Expand Down Expand Up @@ -46,25 +58,69 @@ async def run_for(version: str, runtime: Runtime, lock_manager: LockManager):
runtime.logger.info('[%s] Not permitted, skipping', version)
return

if not await _is_version_okd_enabled(runtime, version):
runtime.logger.info(
'[%s] OKD not enabled in build-data (okd.enabled); skipping okd-scan schedule',
version,
)
return

# Schedule scan
runtime.logger.info('[%s] Scheduling okd-scan', version)
jenkins.start_okd_scan_konflux(version=version, block_until_building=False)


async def resolve_schedule_versions(runtime: Runtime, version: tuple) -> list[str]:
if version:
enabled = []
for candidate in version:
if await _is_version_okd_enabled(runtime, candidate):
enabled.append(candidate)
else:
runtime.logger.info(
'Version %s is not enabled for OKD (set okd.enabled: true in group.yml on openshift-%s). Skipping.',
candidate,
candidate,
)
return enabled
versions = await util.get_okd_enabled_versions(
working_dir=runtime.working_dir / 'doozer_working',
)
if not versions:
runtime.logger.info('No OKD-enabled versions found in build-data; nothing to schedule')
else:
runtime.logger.info('OKD-enabled versions from build-data: %s', ', '.join(versions))
return versions


@cli.command('schedule-okd-scan')
@click.option('--version', '-v', required=True, help='OCP/OKD version to scan', multiple=True)
@click.option(
'--version',
'-v',
required=False,
help='OCP/OKD version to scan (omit to discover enabled versions from build-data)',
multiple=True,
)
@pass_runtime
@click_coroutine
async def okd_scan(runtime: Runtime, version: tuple):
"""
Schedule OKD scans for specified versions.

When --version is omitted, probes ACTIVE_OCP_VERSIONS and schedules only versions
with okd.enabled in group.yml (merged via doozer --variant=okd).

Example usage:
artcd schedule-okd-scan
artcd schedule-okd-scan --version 4.21 --version 4.22
"""
versions = await resolve_schedule_versions(runtime, version)
if not versions:
return

jenkins.init_jenkins()
lock_manager = LockManager([redis.redis_url()])
try:
await asyncio.gather(*[run_for(v, runtime, lock_manager) for v in version])
await asyncio.gather(*[run_for(v, runtime, lock_manager) for v in versions])
finally:
await lock_manager.destroy()
71 changes: 71 additions & 0 deletions pyartcd/pyartcd/util.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import asyncio
import json
import logging
import os
Expand All @@ -16,6 +17,7 @@
from artcommonlib import exectools, redis
from artcommonlib.arch_util import go_suffix_for_arch
from artcommonlib.assembly import assembly_type
from artcommonlib.constants import ACTIVE_OCP_VERSIONS
from artcommonlib.exectools import limit_concurrency
from artcommonlib.github_auth import get_github_client_for_org
from artcommonlib.model import Missing, Model
Expand Down Expand Up @@ -114,6 +116,75 @@ async def load_group_config(
return group_config


def okd_doozer_base_command(
version: str,
assembly: str,
data_path: str,
data_gitref: str,
working_dir: Union[str, Path],
) -> list[str]:
group_param = f'openshift-{version}'
if data_gitref:
group_param += f'@{data_gitref}'
return [
'doozer',
f'--working-dir={working_dir}',
f'--data-path={data_path}',
f'--group={group_param}',
f'--assembly={assembly}',
'--build-system=konflux',
'--variant=okd',
]


async def is_okd_version_enabled(doozer_base_command: list[str]) -> bool:
"""
Check whether OKD is enabled for a version via merged group config.

The doozer command must include --variant=okd so Runtime merges group.yml okd:
into the group config (okd.enabled becomes top-level enabled), matching OKD builds.

Arg(s):
doozer_base_command (list[str]): Base doozer invocation for the target version.
Return Value(s):
bool: True when merged group config has enabled=true.
"""
cmd = [*doozer_base_command, 'config:read-group', 'enabled', '--default=False']
_, out, _ = await exectools.cmd_gather_async(cmd, stderr=None)
return out.strip() == 'True'


async def get_okd_enabled_versions(
assembly: str = 'stream',
data_path: str = constants.OCP_BUILD_DATA_URL,
data_gitref: str = '',
working_dir: Union[str, Path] = '.',
candidates: Optional[List[str]] = None,
) -> List[str]:
"""
Return OCP versions with okd.enabled in merged group config.

Probes each candidate via doozer --variant=okd (same gate as okd-scan).
"""
versions_to_check = candidates if candidates is not None else ACTIVE_OCP_VERSIONS
enabled_versions: List[str] = []

async def check_version(version: str) -> Optional[str]:
base_cmd = okd_doozer_base_command(version, assembly, data_path, data_gitref, working_dir)
if await is_okd_version_enabled(base_cmd):
return version
logger.info(
'Version %s is not enabled for OKD (set okd.enabled: true in group.yml on openshift-%s)',
version,
version,
)
return None

results = await asyncio.gather(*[check_version(version) for version in versions_to_check])
enabled_versions.extend(v for v in results if v)
return enabled_versions


async def load_releases_config(group: str, data_path: str = constants.OCP_BUILD_DATA_URL) -> Optional[Dict]:
cmd = [
'doozer',
Expand Down
Loading
Loading