diff --git a/.github/release/python-tag-filter.txt b/.github/release/python-tag-filter.txt index f9e5b42..a0c4408 100644 --- a/.github/release/python-tag-filter.txt +++ b/.github/release/python-tag-filter.txt @@ -2,3 +2,5 @@ # Examples: # 3.13.* # 3.12.* + +3.11.* diff --git a/.github/scripts/apply_partial_manifests.py b/.github/scripts/apply_partial_manifests.py new file mode 100644 index 0000000..c49d0ac --- /dev/null +++ b/.github/scripts/apply_partial_manifests.py @@ -0,0 +1,89 @@ +import argparse +import json +from pathlib import Path +from typing import Iterable, List, Tuple + +from manifest_tools import update_version + +REQUIRED_FIELDS = {"version", "filename", "arch", "platform", "download_url"} + + +def discover_partial_files(partials_dir: Path) -> List[Path]: + return sorted(partials_dir.rglob("*.json")) + + +def load_entries(file_path: Path) -> List[dict]: + with file_path.open("r", encoding="utf-8") as handle: + data = json.load(handle) + if isinstance(data, list): + return data + return [] + + +def valid_entry(entry: dict) -> bool: + return REQUIRED_FIELDS.issubset(entry.keys()) + + +def ensure_manifest_file(manifest_file: Path) -> None: + manifest_file.parent.mkdir(parents=True, exist_ok=True) + if not manifest_file.exists(): + manifest_file.write_text("[]\n", encoding="utf-8") + + +def apply_entries(entries: Iterable[Tuple[Path, dict]], manifest_dir: Path) -> int: + applied = 0 + for file_path, entry in entries: + if not valid_entry(entry): + continue + manifest_file = manifest_dir / f"{entry['version']}-{entry['arch']}.json" + ensure_manifest_file(manifest_file) + update_version( # type: ignore[arg-type] + existing_file=str(manifest_file), + version=entry["version"], + filename=entry["filename"], + arch=entry["arch"], + platform=entry["platform"], + download_url=entry["download_url"], + platform_version=entry.get("platform_version"), + stable=True, + ) + applied += 1 + print(f"Applied entry from {file_path.name} to {manifest_file}") + return applied + + +def main() -> int: + parser = argparse.ArgumentParser(description="Apply partial manifest artifacts to arch-specific manifests.") + parser.add_argument("--partials-dir", default="manifest-parts", help="Directory containing manifest-part artifacts") + parser.add_argument("--manifest-dir", default="versions-manifests", help="Directory for arch-specific manifests") + args = parser.parse_args() + + partials_path = Path(args.partials_dir) + manifest_path = Path(args.manifest_dir) + + if not partials_path.exists(): + print(f"No partial manifests found in {partials_path}") + return 0 + + files = discover_partial_files(partials_path) + if not files: + print(f"No JSON files discovered under {partials_path}") + return 0 + + entries: List[Tuple[Path, dict]] = [] + for file_path in files: + try: + file_entries = load_entries(file_path) + except json.JSONDecodeError as exc: + print(f"Skipping {file_path}: invalid JSON ({exc})") + continue + for entry in file_entries: + entries.append((file_path, entry)) + + applied = apply_entries(entries, manifest_path) + print(f"Applied {applied} manifest entries from {len(files)} partial files.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/generate_partial_manifest.py b/.github/scripts/generate_partial_manifest.py new file mode 100644 index 0000000..b6dbf2d --- /dev/null +++ b/.github/scripts/generate_partial_manifest.py @@ -0,0 +1,160 @@ +import argparse +import json +import sys +from typing import Any, Dict, List, Optional, Tuple + +PREFIXES = ("python-", "trivy-python-") +SUFFIXES = (".tar.gz", ".sbom.json", ".json", ".log") + + +def validate_download_url(url: str, owner: str, repo: str, tag: str, filename: str) -> bool: + """Validate that download_url is well-formed and points to a release asset. + + Note: GitHub may initially serve assets via temporary "untagged" URLs. + We validate the basic structure and accept it, knowing we'll construct + the final URL ourselves to avoid issues with ephemeral URLs. + """ + if not url or not url.strip(): + return False + + # Must be HTTPS and from GitHub releases + if not url.startswith("https://github.com/"): + return False + + # Must be from the correct owner/repo releases + expected_base = f"https://github.com/{owner}/{repo}/releases/download/" + if not url.startswith(expected_base): + return False + + # URL structure is valid - we'll construct the final URL ourselves + # to avoid issues with temporary "untagged-" URLs from GitHub + return True + + +def strip_known_wrappers(filename: str) -> str: + """Remove known prefixes and suffixes from the asset name.""" + cleaned = filename + for prefix in PREFIXES: + if cleaned.startswith(prefix): + cleaned = cleaned[len(prefix) :] + break + for suffix in SUFFIXES: + if cleaned.endswith(suffix): + cleaned = cleaned[: -len(suffix)] + break + return cleaned + + +def parse_filename(filename: str) -> Optional[Dict[str, str]]: + """Extract platform metadata from a release asset filename.""" + stripped = strip_known_wrappers(filename) + parts = stripped.split("-") + if len(parts) < 4: + return None + + return { + "version": parts[0], + "platform": parts[1], + "platform_version": parts[2], + "arch": parts[3], + } + + +def should_skip(name: str) -> bool: + return not name.endswith(".tar.gz") or "trivy" in name + + +def build_manifest_entries(tag: str, assets: List[Dict[str, Any]], owner: str, repo: str) -> Tuple[List[Dict[str, str]], List[str]]: + """Build manifest entries and return (entries, validation_errors).""" + entries: List[Dict[str, str]] = [] + errors: List[str] = [] + + for asset in assets: + name = asset.get("name", "") + if should_skip(name): + continue + + parsed = parse_filename(name) + if not parsed: + errors.append(f"Filename parsing failed for asset '{name}'") + continue + + download_url = asset.get("browser_download_url", "") + + # Validate URL structure (basic sanity check) + if not validate_download_url(download_url, owner, repo, tag, name): + errors.append( + f"Invalid download_url for '{name}': expected " + f"'https://github.com/{owner}/{repo}/releases/download/{tag}/{name}', " + f"got '{download_url}'" + ) + continue + + # Construct the permanent, tagged download URL to avoid "untagged-*" ephemeral URLs + # GitHub may initially serve assets via temporary URLs, so we construct the final one + final_download_url = f"https://github.com/{owner}/{repo}/releases/download/{tag}/{name}" + + entries.append( + { + "version": tag, + "filename": name, + "arch": parsed["arch"], + "platform": parsed["platform"], + "platform_version": parsed["platform_version"], + "download_url": final_download_url, + } + ) + + return entries, errors + + +def main() -> int: + parser = argparse.ArgumentParser(description="Generate partial manifest entries from release assets.") + parser.add_argument("--tag", required=True, help="Release tag used for the manifest version field.") + parser.add_argument("--owner", required=True, help="GitHub repository owner (e.g., 'IBM').") + parser.add_argument("--repo", required=True, help="GitHub repository name (e.g., 'python-versions-pz').") + + # Mutually exclusive group: Accept EITHER string OR file + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--assets", help="Raw JSON string (Legacy/Direct Input)") + group.add_argument("--assets-file", help="Path to JSON file containing assets (Recommended for CI)") + + args = parser.parse_args() + + # Logic to load assets from either source + assets = [] + try: + if args.assets_file: + print(f"Reading assets from file: {args.assets_file}", file=sys.stderr) + with open(args.assets_file, 'r', encoding='utf-8') as f: + assets = json.load(f) + elif args.assets: + # Legacy support for workflows passing raw strings + print(f"Parsing assets from command-line string", file=sys.stderr) + assets = json.loads(args.assets) + except json.JSONDecodeError as exc: + print(f"Error decoding JSON: {exc}", file=sys.stderr) + return 1 + except FileNotFoundError as exc: + print(f"Assets file not found: {args.assets_file}", file=sys.stderr) + return 1 + + manifest_entries, errors = build_manifest_entries(args.tag, assets, args.owner, args.repo) + + # Report validation errors + if errors: + for error in errors: + print(f"Validation error: {error}", file=sys.stderr) + if not manifest_entries: + print(f"ERROR: No valid assets found after validation. Aborting.", file=sys.stderr) + return 1 + # Warn but continue if some assets are valid + print(f"Warning: {len(errors)} asset(s) failed validation but {len(manifest_entries)} remain.", file=sys.stderr) + + json.dump(manifest_entries, sys.stdout, indent=2) + sys.stdout.write("\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/backfill-manifests.yml b/.github/workflows/backfill-manifests.yml new file mode 100644 index 0000000..5dd8bd7 --- /dev/null +++ b/.github/workflows/backfill-manifests.yml @@ -0,0 +1,184 @@ +name: Backfill Manifests + +on: + workflow_dispatch: + inputs: + limit: + description: "Number of releases to process (Max 250 due to GHA matrix limits)" + required: false + default: "30" + type: string + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + get-all-tags: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Fetch release tags + id: set-matrix + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + LIMIT: ${{ inputs.limit }} + run: | + # Safety Check: Matrix strategy has a hard limit of 256 jobs + if [ "$LIMIT" -gt 250 ]; then + echo "::warning::Limit $LIMIT exceeds GitHub Matrix safety. Capping at 250." + LIMIT=250 + fi + + echo "Fetching last $LIMIT releases..." + + TAGS=$(gh release list \ + --limit "$LIMIT" \ + --exclude-drafts \ + --exclude-pre-releases \ + --json tagName \ + --jq '[.[] | .tagName]') + + if [ -z "$TAGS" ] || [ "$TAGS" == "[]" ]; then + echo "::error::No release tags found!" + exit 1 + fi + + echo "Found tags count: $(echo $TAGS | jq '. | length')" + echo "matrix=$TAGS" >> $GITHUB_OUTPUT + + generate-manifests: + needs: get-all-tags + runs-on: ubuntu-latest + strategy: + fail-fast: false + max-parallel: 10 # Slightly higher parallelization is fine for reads + matrix: + tag: ${{ fromJson(needs.get-all-tags.outputs.matrix) }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install poetry + poetry install --no-interaction --no-ansi + + - name: Fetch Assets Data + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "::group::Fetch Assets via GitHub CLI" + # Fetch assets with all required fields including browser_download_url + gh api "repos/${{ github.repository }}/releases/tags/${{ matrix.tag }}" \ + --jq '.assets | map({name: .name, browser_download_url: .browser_download_url})' \ + > assets.json + + if [ ! -s assets.json ]; then + echo "::error::Assets file is empty for tag ${{ matrix.tag }}" + exit 1 + fi + + echo "Assets fetched:" + cat assets.json + echo "::endgroup::" + + - name: Generate Manifest + env: + OWNER: ${{ github.repository_owner }} + run: | + echo "::group::Run Generation Script" + + REPO_NAME="${{ github.repository }}" && REPO_NAME="${REPO_NAME##*/}" + + # We use --assets-file now instead of --assets + poetry run python .github/scripts/generate_partial_manifest.py \ + --tag "${{ matrix.tag }}" \ + --owner "${{ github.repository_owner }}" \ + --repo "${REPO_NAME}" \ + --assets-file "assets.json" \ + > manifest-part-${{ matrix.tag }}.json + + cat manifest-part-${{ matrix.tag }}.json + echo "::endgroup::" + + - name: Upload Manifest Artifact + uses: actions/upload-artifact@v4 + with: + name: manifest-part-${{ matrix.tag }} + path: manifest-part-${{ matrix.tag }}.json + retention-days: 1 + + apply-manifests: + needs: generate-manifests + if: always() && needs.generate-manifests.result != 'skipped' && needs.generate-manifests.result != 'cancelled' + runs-on: ubuntu-latest + permissions: + contents: write + actions: read + concurrency: + group: manifests-backfill-${{ github.ref }} + cancel-in-progress: false + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Download partial manifest artifacts + continue-on-error: true + uses: actions/download-artifact@v4 + with: + path: manifest-parts + pattern: manifest-part-* + merge-multiple: true + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + python -m pip install poetry + poetry install --no-interaction --no-ansi + + - name: Apply partial manifests + run: | + poetry run python .github/scripts/apply_partial_manifests.py \ + --partials-dir manifest-parts \ + --manifest-dir versions-manifests + + - name: Commit manifest updates + id: commit_manifests + run: | + if git status --porcelain -- versions-manifests | grep .; then + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add versions-manifests/*.json + git commit -s -m "Backfill manifests from workflow run [skip ci]" + echo "committed=true" >> "$GITHUB_OUTPUT" + else + echo "No manifest changes detected" + echo "committed=false" >> "$GITHUB_OUTPUT" + fi + + - name: Push manifest updates + if: steps.commit_manifests.outputs.committed == 'true' + run: | + git pull --rebase --strategy=recursive --strategy-option=ours + git push diff --git a/.github/workflows/generate_tar.yml b/.github/workflows/generate_tar.yml index 880932a..de8fc9b 100644 --- a/.github/workflows/generate_tar.yml +++ b/.github/workflows/generate_tar.yml @@ -200,7 +200,7 @@ jobs: git add -- "${f}" done if ! git diff --cached --quiet; then - git commit -m "Add/update powershell-gen tar for ${{ steps.get_tag_extract.outputs.pwsh_tag }}" + git commit -s -m "Add/update powershell-gen tar for ${{ steps.get_tag_extract.outputs.pwsh_tag }}" git push "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" "HEAD:${GITHUB_REF#refs/heads/}" else echo "No changes to commit." diff --git a/.github/workflows/merge-manifest.yml b/.github/workflows/merge-manifest.yml index 35ac639..82bc948 100644 --- a/.github/workflows/merge-manifest.yml +++ b/.github/workflows/merge-manifest.yml @@ -55,5 +55,5 @@ jobs: - name: Commit and push main manifest run: | git add versions-manifest.json - git commit -m "Update main manifest [skip ci]" || echo "No changes to commit" + git commit -s -m "Update main manifest [skip ci]" || echo "No changes to commit" git push \ No newline at end of file diff --git a/.github/workflows/python-sample.yml b/.github/workflows/python-sample.yml index cc784af..ed34825 100644 --- a/.github/workflows/python-sample.yml +++ b/.github/workflows/python-sample.yml @@ -25,7 +25,7 @@ jobs: - name: Set up Python uses: IBM/setup-python-pz@main with: - python-version: 3.13.4 + python-version: 3.14 architecture: ${{ matrix.arch }} - name: Test Python installation diff --git a/.github/workflows/release-latest-python-tag.yml b/.github/workflows/release-latest-python-tag.yml index e10c811..5f2d34b 100644 --- a/.github/workflows/release-latest-python-tag.yml +++ b/.github/workflows/release-latest-python-tag.yml @@ -12,6 +12,11 @@ permissions: contents: read # Required for checkout and reading repository content actions: write # Required for calling reusable workflows +# Keep only one release-latest run per ref to avoid overlapping build/release pipelines. +concurrency: + group: release-latest-${{ github.ref }} + cancel-in-progress: false + jobs: get-latest-tag: runs-on: ubuntu-latest @@ -50,25 +55,88 @@ jobs: build-and-release-matrix: needs: get-latest-tag strategy: + fail-fast: false # Allow other legs to proceed even if one fails matrix: + arch: ['ppc64le', 's390x'] platform-version: ['24.04', '22.04'] - include: - - arch: ppc64le - runner-label: ubuntu-24.04-ppc64le - - arch: s390x - runner-label: ubuntu-24.04-s390x uses: ./.github/workflows/reusable-build-and-release-python-versions.yml with: arch: ${{ matrix.arch }} tag: ${{ needs.get-latest-tag.outputs.latest_tag }} platform-version: ${{ matrix.platform-version }} - runner-label: ${{ matrix.runner-label }} + runner-label: ${{ format('ubuntu-{0}-{1}', matrix['platform-version'], matrix.arch) }} release-asset: needs: [build-and-release-matrix, get-latest-tag] + if: always() && !cancelled() + permissions: + contents: write + actions: read uses: ./.github/workflows/reusable-release-python-tar.yml with: tag: ${{ needs.get-latest-tag.outputs.latest_tag }} secrets: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + update-manifests: + needs: release-asset + if: always() && needs.release-asset.result != 'skipped' && needs.release-asset.result != 'cancelled' + runs-on: ubuntu-latest + permissions: + contents: write + actions: read + concurrency: + group: manifests-${{ github.ref }} # Shared with other workflows so only one manifest push happens at a time + cancel-in-progress: false + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Download partial manifest artifacts # Pull the short-lived JSON blobs produced by the release job + continue-on-error: true # Missing artifacts just mean no new manifests + uses: actions/download-artifact@v4 + with: + path: manifest-parts + pattern: manifest-part-* + merge-multiple: true + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install poetry + poetry install --no-interaction --no-ansi + + - name: Apply partial manifests # Converts partials into tracked versions-manifests/*.json files + run: | + poetry run python .github/scripts/apply_partial_manifests.py \ + --partials-dir manifest-parts \ + --manifest-dir versions-manifests + + - name: Commit manifest updates + id: commit_manifests + run: | + if git status --porcelain -- versions-manifests | grep .; then + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add versions-manifests/*.json + git commit -s -m "Apply manifest partials [skip ci]" + echo "committed=true" >> "$GITHUB_OUTPUT" + else + echo "No manifest changes detected" + echo "committed=false" >> "$GITHUB_OUTPUT" + fi + + - name: Push manifest updates + if: steps.commit_manifests.outputs.committed == 'true' + run: | + # Keep freshly generated manifest files if a conflict arises + git pull --rebase --strategy=recursive --strategy-option=ours + git push + diff --git a/.github/workflows/release-matching-python-tags.yml b/.github/workflows/release-matching-python-tags.yml index 1880ebe..f928b6a 100644 --- a/.github/workflows/release-matching-python-tags.yml +++ b/.github/workflows/release-matching-python-tags.yml @@ -7,11 +7,19 @@ name: Release Matching Python Tags on: # Inputs must be empty to comply with policy workflow_dispatch: {} + push: + paths: + - .github/release/python-tag-filter.txt permissions: contents: write # Required for release workflow to push manifest and assets actions: write # Required for calling reusable workflows +# Ensure only one release-matching run per ref executes at a time (builds are expensive). +concurrency: + group: release-matching-${{ github.ref }} + cancel-in-progress: false + jobs: get-tags: runs-on: ubuntu-latest @@ -67,29 +75,28 @@ jobs: build-and-release-matrix: needs: get-tags strategy: + fail-fast: false # Allow other builds to continue even if a sibling fails + max-parallel: 8 # Reduce concurrent jobs to limit load on runners matrix: tag: ${{ fromJson(needs.get-tags.outputs.tags_json) }} platform-version: ['24.04', '22.04'] arch: ['s390x', 'ppc64le'] - include: - - arch: s390x - runner-label: ubuntu-24.04-s390x - - arch: ppc64le - runner-label: ubuntu-24.04-ppc64le uses: ./.github/workflows/reusable-build-and-release-python-versions.yml with: arch: ${{ matrix.arch }} tag: ${{ matrix.tag }} platform-version: ${{ matrix['platform-version'] }} - runner-label: ${{ matrix['runner-label'] }} + runner-label: ${{ format('ubuntu-{0}-{1}', matrix['platform-version'], matrix.arch) }} release-assets: needs: [build-and-release-matrix, get-tags] + if: always() && !cancelled() permissions: contents: write actions: read strategy: + fail-fast: false matrix: tag: ${{ fromJson(needs.get-tags.outputs.tags_json) }} uses: ./.github/workflows/reusable-release-python-tar.yml @@ -97,3 +104,65 @@ jobs: tag: ${{ matrix.tag }} secrets: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + update-manifests: + needs: release-assets + if: always() && needs.release-assets.result != 'skipped' && needs.release-assets.result != 'cancelled' + runs-on: ubuntu-latest + permissions: + contents: write + actions: read + concurrency: + group: manifests-${{ github.ref }} # Shared with release-latest so manifest pushes serialize + cancel-in-progress: false + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Download partial manifest artifacts # Pull manifest-part-* artifacts from successful release jobs + continue-on-error: true # Some matrix legs may fail and omit artifacts + uses: actions/download-artifact@v4 + with: + path: manifest-parts + pattern: manifest-part-* + merge-multiple: true + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install poetry + poetry install --no-interaction --no-ansi + + - name: Apply partial manifests # Convert partial JSON blobs into tracked versions-manifests/*.json + run: | + poetry run python .github/scripts/apply_partial_manifests.py \ + --partials-dir manifest-parts \ + --manifest-dir versions-manifests + + - name: Commit manifest updates + id: commit_manifests + run: | + if git status --porcelain -- versions-manifests | grep .; then + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add versions-manifests/*.json + git commit -s -m "Apply manifest partials [skip ci]" + echo "committed=true" >> "$GITHUB_OUTPUT" + else + echo "No manifest changes detected" + echo "committed=false" >> "$GITHUB_OUTPUT" + fi + + - name: Push manifest updates + if: steps.commit_manifests.outputs.committed == 'true' + run: | + # Keep locally generated manifest files when resolving conflicts + git pull --rebase --strategy=recursive --strategy-option=ours + git push diff --git a/.github/workflows/reusable-release-python-tar.yml b/.github/workflows/reusable-release-python-tar.yml index 94be4a7..c03b147 100644 --- a/.github/workflows/reusable-release-python-tar.yml +++ b/.github/workflows/reusable-release-python-tar.yml @@ -8,7 +8,7 @@ on: required: true type: string files: - description: "Glob pattern(s) for release files passed to softprops/action-gh-release (newline separated)" + description: "Glob pattern(s) for release files" required: false type: string default: | @@ -20,13 +20,16 @@ on: required: true permissions: - contents: write # Required for creating releases and pushing commits - actions: read # Required for downloading artifacts + contents: write # Required for creating releases + actions: read # Required for downloading artifacts jobs: - release-assets-and-update-manifest: + release-assets-and-generate-json: runs-on: ubuntu-latest steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Download all build artifacts uses: actions/download-artifact@v4 with: @@ -37,25 +40,16 @@ jobs: - name: List downloaded files (debugging step) run: ls -R ./artifact - - name: Release all .tar.gz files from python-tar + - name: Release all .tar.gz files id: create_release uses: softprops/action-gh-release@v2 with: tag_name: ${{ inputs.tag }} name: Release ${{ inputs.tag }} - files: ${{ inputs.files }} # Accept a files pattern from the calling workflow, default set on workflow_call + files: ${{ inputs.files }} draft: false prerelease: false - token: ${{ secrets.GH_TOKEN }} # Use the token passed from the calling workflow - - - name: Checkout repository - uses: actions/checkout@v4 - with: - token: ${{ secrets.GH_TOKEN }} # Token needed for push operations - - - name: Pull latest changes - # Ensure we have the latest state of the repo, especially the versions-manifests folder - run: git pull + token: ${{ secrets.GH_TOKEN }} - name: Set up Python uses: actions/setup-python@v5 @@ -67,142 +61,26 @@ jobs: python -m pip install --upgrade pip pip install poetry poetry install --no-interaction --no-ansi - working-directory: . - - name: Create, commit, and push arch-specific manifest for released asset + - name: Generate partial manifest + id: generate_manifest env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} + ASSETS_JSON: ${{ steps.create_release.outputs.assets }} run: | - assets_json='${{ steps.create_release.outputs.assets }}' - asset_count=$(echo "$assets_json" | jq length) - - # Use a Bash associative array to group assets by their architecture - declare -A ARCH_ASSETS_MAP - - # First pass: Group asset indices by unique architecture from the downloaded files - # This handles cases where a single 'tag' might have been built for multiple platform-versions - # (e.g., 3.13.3-linux-22.04-ppc64le.tar.gz and 3.13.3-linux-24.04-ppc64le.tar.gz) - # within the same 'arch'. - for i in $(seq 0 $((asset_count - 1))); do - asset_name=$(echo "$assets_json" | jq -r ".[$i].name") - # Normalize and strip known prefixes and extensions we expect - stripped="$asset_name" - # Remove common prefixes 'python-' and 'trivy-python-' (for security artifacts) - stripped="${stripped#python-}" - stripped="${stripped#trivy-python-}" - # Remove several known extensions - stripped="${stripped%.tar.gz}" - stripped="${stripped%.sbom.json}" - stripped="${stripped%.json}" - stripped="${stripped%.log}" - - # Split the stripped name into parts; expected format: ---[-optional] - IFS='-' read -r -a parts <<< "$stripped" - # Only group assets that include an architecture token at the expected position - if [ ${#parts[@]} -ge 4 ]; then - current_arch="${parts[3]}" - ARCH_ASSETS_MAP["$current_arch"]+=" $i" - fi - done - - # Ensure the dedicated manifests directory exists - mkdir -p versions-manifests - - # Second pass: Process each unique architecture found and build its manifest file - # We iterate over the architectures identified from the downloaded artifacts - for arch_key in "${!ARCH_ASSETS_MAP[@]}"; do - MANIFEST_FILE="versions-manifests/${{ inputs.tag }}-${arch_key}.json" - - echo "::group::Processing manifest for tag ${{ inputs.tag }}, arch ${arch_key}" - - # Initialize the manifest file for this specific tag-arch combination. - # This will create a new empty file. If it already exists from a previous run, it will be overwritten. - echo "[]" > "$MANIFEST_FILE" - - # Iterate over asset indices that belong to this specific architecture - for asset_idx in ${ARCH_ASSETS_MAP["$arch_key"]}; do - asset_name=$(echo "$assets_json" | jq -r ".[$asset_idx].name") - download_url=$(echo "$assets_json" | jq -r ".[$asset_idx].browser_download_url") - - # Normalize and strip known prefixes and extensions - stripped="$asset_name" - stripped="${stripped#python-}" - stripped="${stripped#trivy-python-}" - stripped="${stripped%.tar.gz}" - stripped="${stripped%.sbom.json}" - stripped="${stripped%.json}" - stripped="${stripped%.log}" - IFS='-' read -r -a parts <<< "$stripped" - # Determine fields: version, platform, platform_version, arch - platform="${parts[1]:-}" || true - platform_version="${parts[2]:-}" || true - actual_arch="${parts[3]:-}" - - echo " - Asset: $asset_name" - echo " Platform: $platform" - echo " Platform version: $platform_version" - echo " Download URL: $download_url" - - # Use your Python script to update the specific manifest file - poetry run python .github/scripts/manifest_tools.py update_version \ - "$MANIFEST_FILE" \ - --version "${{ inputs.tag }}" \ - --filename "$asset_name" \ - --arch "$actual_arch" \ - --platform "$platform" \ - --platform-version "$platform_version" \ - --download-url "$download_url" \ - --stable - done - echo "::endgroup::" - done - - # Final step: Commit and push ALL modified arch-specific manifest files in one atomic Git operation. - # This is done once after all unique arch manifests for this run have been prepared. - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - - # Add all JSON files within the versions-manifests directory that were modified - git add versions-manifests/*.json - - # Only commit if there are actual changes - git diff --staged --quiet || git commit -m "Update arch manifests for tag ${{ inputs.tag }} [skip ci]" - - # Rebase before pushing to gracefully handle any concurrent pushes from other workflow runs - # that might have pushed different arch-specific manifests in parallel. - git pull --rebase - # Exponential back-off with jitter for push - max_retries=20 # Increased retries for higher concurrency - base_delay=1 # Starting delay in seconds - max_delay=90 # Maximum delay in seconds - count=0 - - until git push; do - count=$((count + 1)) - if [ $count -ge $max_retries ]; then - echo "Push failed after $max_retries attempts. Exiting with failure." - exit 1 - fi - - current_delay=$(( base_delay * (2**(count-1)) )) - # Cap the delay at max_delay - if [ "$current_delay" -gt "$max_delay" ]; then - current_delay="$max_delay" - fi - - # Add jitter: +/- 25% of the current_delay (but at least 1s) - jitter_range=$(( current_delay / 4 )) - random_jitter=$(( RANDOM % (2 * jitter_range + 1) - jitter_range )) - sleep_duration=$(( current_delay + random_jitter )) - - # Ensure sleep_duration is at least 1 second to avoid hammering the server - if [ "$sleep_duration" -lt 1 ]; then - sleep_duration=1 - fi - - echo "Push attempt failed. Attempting to fetch, rebase, and retry ($count/$max_retries). Waiting ${sleep_duration}s..." - # Always rebase against the target branch to incorporate others' distinct commits - git fetch origin - git pull --rebase origin ${{ github.ref_name }} - sleep "$sleep_duration" - done + REPO_NAME="${{ github.repository }}" && REPO_NAME="${REPO_NAME##*/}" + poetry run python .github/scripts/generate_partial_manifest.py \ + --tag "${{ inputs.tag }}" \ + --owner "${{ github.repository_owner }}" \ + --repo "${REPO_NAME}" \ + --assets "$ASSETS_JSON" \ + > manifest-part-${{ inputs.tag }}.json + + echo "Generated Manifest Content:" + cat manifest-part-${{ inputs.tag }}.json + + - name: Upload partial manifest artifact + uses: actions/upload-artifact@v4 + with: + name: manifest-part-${{ inputs.tag }} + path: manifest-part-${{ inputs.tag }}.json + retention-days: 1 diff --git a/Makefile b/Makefile index 3aacf18..04eadf1 100644 --- a/Makefile +++ b/Makefile @@ -21,11 +21,11 @@ ACTIONS_PYTHON_VERSIONS ?= 3.13.3-14344076652 POWERSHELL_VERSION ?= v7.5.2 POWERSHELL_NATIVE_VERSION ?= v7.4.0 UBUNTU_VERSION ?= 24.04 -TRIVY_VERSION ?= v0.58.1 +TRIVY_VERSION ?= v0.68.2 # Security Gates (0 = Log Only, 1 = Fail Build) -FAIL_ON_CRITICAL ?= 0 -FAIL_ON_HIGH ?= 0 +FAIL_ON_CRITICAL ?= 1 +FAIL_ON_HIGH ?= 1 FAIL_ON_MEDIUM ?= 0 FAIL_ON_SECRET ?= 0 diff --git a/PowerShell/dotnet-install.py b/PowerShell/dotnet-install.py index a0bac28..daecc62 100644 --- a/PowerShell/dotnet-install.py +++ b/PowerShell/dotnet-install.py @@ -14,7 +14,9 @@ import shutil import json import urllib.request +import urllib.error import bisect +import time from typing import Optional, List, Tuple, NamedTuple # Third-party imports @@ -25,6 +27,8 @@ GH_REPO = "dotnet-s390x" INSTALL_DIR = "/usr/share/dotnet" NUGET_PACKAGE = "microsoft.netcore.app.runtime.linux-x64" +FETCH_MAX_RETRIES = 8 +FETCH_RETRY_DELAY = 5 app = typer.Typer() @@ -172,11 +176,25 @@ def find_closest_version_tag(all_tags: List[dict], input_tag: str) -> str: PROFILE_SCRIPT = "/etc/profile.d/dotnet.sh" def fetch_json(url: str) -> List[dict]: - """Download and parse JSON response from a given URL.""" - with urllib.request.urlopen(url) as response: - if response.status >= 400: - raise typer.Exit(f"❌ Failed to fetch {url}") - return json.loads(response.read()) + """Download and parse JSON response from a given URL with basic retries.""" + for attempt in range(FETCH_MAX_RETRIES): + try: + with urllib.request.urlopen(url) as response: + if response.status >= 400: + raise typer.Exit(f"❌ Failed to fetch {url}") + return json.loads(response.read()) + except urllib.error.HTTPError as exc: # Retry transient HTTP errors + if exc.code in [500, 502, 503, 504] and attempt < FETCH_MAX_RETRIES - 1: + typer.echo(f"⚠️ HTTP {exc.code} fetching {url}. Retrying in {FETCH_RETRY_DELAY}s... ({attempt + 1}/{FETCH_MAX_RETRIES})") + time.sleep(FETCH_RETRY_DELAY) + continue + raise + except Exception as exc: + if attempt < FETCH_MAX_RETRIES - 1: + typer.echo(f"⚠️ Error fetching {url}: {exc}. Retrying in {FETCH_RETRY_DELAY}s... ({attempt + 1}/{FETCH_MAX_RETRIES})") + time.sleep(FETCH_RETRY_DELAY) + continue + raise def get_all_tags() -> List[dict]: """Fetch all release tags from the IBM GitHub repository.""" diff --git a/tests/test_apply_partial_manifests.py b/tests/test_apply_partial_manifests.py new file mode 100644 index 0000000..bd17680 --- /dev/null +++ b/tests/test_apply_partial_manifests.py @@ -0,0 +1,88 @@ +import json +import sys +from pathlib import Path + +import apply_partial_manifests as apm + + +def write_partial(base_path: Path, name: str, entries): + folder = base_path / name + folder.mkdir(parents=True, exist_ok=True) + file_path = folder / f"{name}.json" + file_path.write_text(json.dumps(entries), encoding="utf-8") + return file_path + + +def test_apply_partial_manifests_creates_manifest(tmp_path, monkeypatch): + partial_dir = tmp_path / "partials" + manifest_dir = tmp_path / "manifests" + partial_dir.mkdir() + manifest_dir.mkdir() + + sample_entry = { + "version": "3.13.3", + "filename": "python-3.13.3-linux-22.04-ppc64le.tar.gz", + "arch": "ppc64le", + "platform": "linux", + "platform_version": "22.04", + "download_url": "https://example.com/python.tar.gz", + } + created = write_partial(partial_dir, "manifest-part-3.13.3", [sample_entry]) + assert created.exists(), "Partial artifact fixture failed" + + args = [ + "apply_partial_manifests.py", + "--partials-dir", + str(partial_dir), + "--manifest-dir", + str(manifest_dir), + ] + monkeypatch.setattr(sys, "argv", args) + + exit_code = apm.main() + assert exit_code == 0 + + manifest_file = manifest_dir / "3.13.3-ppc64le.json" + assert manifest_file.exists() + data = json.loads(manifest_file.read_text(encoding="utf-8")) + assert data[0]["files"][0]["filename"].endswith("ppc64le.tar.gz") + + +def test_apply_partial_manifests_ignores_invalid_files(tmp_path, monkeypatch): + partial_dir = tmp_path / "partials" + manifest_dir = tmp_path / "manifests" + partial_dir.mkdir() + manifest_dir.mkdir() + + write_partial(partial_dir, "manifest-part-bad", {"foo": "bar"}) + + args = [ + "apply_partial_manifests.py", + "--partials-dir", + str(partial_dir), + "--manifest-dir", + str(manifest_dir), + ] + monkeypatch.setattr(sys, "argv", args) + + exit_code = apm.main() + assert exit_code == 0 + assert not list(manifest_dir.glob("*.json")) + + +def test_apply_partial_manifests_no_partials(tmp_path, monkeypatch): + partial_dir = tmp_path / "partials" + manifest_dir = tmp_path / "manifests" + # Intentionally do not create partial_dir + + args = [ + "apply_partial_manifests.py", + "--partials-dir", + str(partial_dir), + "--manifest-dir", + str(manifest_dir), + ] + monkeypatch.setattr(sys, "argv", args) + + exit_code = apm.main() + assert exit_code == 0 diff --git a/tests/test_generate_partial_manifest.py b/tests/test_generate_partial_manifest.py new file mode 100644 index 0000000..4b185bd --- /dev/null +++ b/tests/test_generate_partial_manifest.py @@ -0,0 +1,219 @@ +import json +import sys + +import pytest + +import generate_partial_manifest as gpm + + +def test_validate_download_url_valid(): + """Test URL validation with correct URL.""" + url = "https://github.com/IBM/python-versions-pz/releases/download/3.13.3/python-3.13.3-linux-22.04-ppc64le.tar.gz" + assert gpm.validate_download_url(url, "IBM", "python-versions-pz", "3.13.3", "python-3.13.3-linux-22.04-ppc64le.tar.gz") + + +def test_validate_download_url_empty(): + """Test URL validation with empty URL.""" + assert not gpm.validate_download_url("", "IBM", "python-versions-pz", "3.13.3", "python.tar.gz") + + +def test_validate_download_url_untagged_release(): + """Test that untagged releases are accepted (GitHub temporarily serves these). + + The validate function only checks basic structure. The final URL is constructed + in build_manifest_entries() to avoid ephemeral 'untagged' URLs. + """ + url = "https://github.com/IBM/python-versions-pz/releases/download/untagged-abc123/python-3.13.3-linux-22.04-ppc64le.tar.gz" + # Untagged URLs are accepted at validation level + assert gpm.validate_download_url(url, "IBM", "python-versions-pz", "3.13.3", "python-3.13.3-linux-22.04-ppc64le.tar.gz") + + +def test_validate_download_url_wrong_owner(): + """Test URL validation with wrong owner.""" + url = "https://github.com/WRONG/python-versions-pz/releases/download/3.13.3/python-3.13.3-linux-22.04-ppc64le.tar.gz" + assert not gpm.validate_download_url(url, "IBM", "python-versions-pz", "3.13.3", "python-3.13.3-linux-22.04-ppc64le.tar.gz") + + +def test_validate_download_url_wrong_tag(): + """Test URL validation with wrong tag. + + Note: validate_download_url only checks basic structure (HTTPS, owner, repo). + The final URL with correct tag is constructed in build_manifest_entries(). + """ + url = "https://github.com/IBM/python-versions-pz/releases/download/3.12.0/python-3.13.3-linux-22.04-ppc64le.tar.gz" + # At validation level, URL structure is accepted (will be reconstructed later) + assert gpm.validate_download_url(url, "IBM", "python-versions-pz", "3.13.3", "python-3.13.3-linux-22.04-ppc64le.tar.gz") + + +def test_validate_download_url_wrong_filename(): + """Test URL validation with mismatched filename. + + Note: validate_download_url only checks basic structure (HTTPS, owner, repo). + The filename is used to construct the final URL in build_manifest_entries(). + """ + url = "https://github.com/IBM/python-versions-pz/releases/download/3.13.3/python-3.12.0-linux-22.04-ppc64le.tar.gz" + # At validation level, URL structure is accepted (will be reconstructed with correct filename) + assert gpm.validate_download_url(url, "IBM", "python-versions-pz", "3.13.3", "python-3.13.3-linux-22.04-ppc64le.tar.gz") + + +def test_parse_filename_valid_asset(): + name = "python-3.13.3-linux-22.04-ppc64le.tar.gz" + parsed = gpm.parse_filename(name) + assert parsed == { + "version": "3.13.3", + "platform": "linux", + "platform_version": "22.04", + "arch": "ppc64le", + } + + +def test_parse_filename_invalid_asset(): + assert gpm.parse_filename("python-3.13.3-linux.tar.gz") is None + + +def test_build_manifest_entries_filters_assets(): + assets = [ + { + "name": "python-3.13.3-linux-22.04-ppc64le.tar.gz", + "browser_download_url": "https://github.com/IBM/python-versions-pz/releases/download/3.13.3/python-3.13.3-linux-22.04-ppc64le.tar.gz", + }, + { + "name": "trivy-python-3.13.3-linux-22.04-ppc64le.tar.gz", + "browser_download_url": "https://example.com/trivy.tar.gz", + }, + { + "name": "python-3.13.3-linux-22.04-ppc64le.log", + "browser_download_url": "https://example.com/python.log", + }, + ] + + entries, errors = gpm.build_manifest_entries("3.13.3", assets, "IBM", "python-versions-pz") + assert len(entries) == 1 + assert entries[0]["filename"].endswith("tar.gz") + assert entries[0]["arch"] == "ppc64le" + assert len(errors) == 0 # Valid URLs should have no errors + + +def test_build_manifest_entries_invalid_urls(): + """Test that URLs from wrong owner/repo are still rejected.""" + assets = [ + { + "name": "python-3.13.3-linux-22.04-ppc64le.tar.gz", + "browser_download_url": "https://github.com/WRONG-OWNER/python-versions-pz/releases/download/3.13.3/python-3.13.3-linux-22.04-ppc64le.tar.gz", + }, + ] + + entries, errors = gpm.build_manifest_entries("3.13.3", assets, "IBM", "python-versions-pz") + assert len(entries) == 0 + assert len(errors) > 0 + assert "Invalid download_url" in errors[0] + + +def test_main_outputs_manifest(monkeypatch, capsys): + assets = [ + { + "name": "python-3.13.3-linux-22.04-ppc64le.tar.gz", + "browser_download_url": "https://github.com/IBM/python-versions-pz/releases/download/3.13.3/python-3.13.3-linux-22.04-ppc64le.tar.gz", + } + ] + args = [ + "generate_partial_manifest.py", + "--tag", + "3.13.3", + "--owner", + "IBM", + "--repo", + "python-versions-pz", + "--assets", + json.dumps(assets), + ] + monkeypatch.setattr(sys, "argv", args) + + exit_code = gpm.main() + assert exit_code == 0 + + captured = capsys.readouterr() + payload = json.loads(captured.out) + assert payload[0]["download_url"].endswith("python-3.13.3-linux-22.04-ppc64le.tar.gz") + + +def test_main_outputs_manifest_from_file(monkeypatch, capsys, tmp_path): + """Test that main() reads from --assets-file correctly.""" + assets = [ + { + "name": "python-3.13.3-linux-22.04-ppc64le.tar.gz", + "browser_download_url": "https://github.com/IBM/python-versions-pz/releases/download/3.13.3/python-3.13.3-linux-22.04-ppc64le.tar.gz", + } + ] + + assets_file = tmp_path / "assets.json" + assets_file.write_text(json.dumps(assets), encoding='utf-8') + + args = [ + "generate_partial_manifest.py", + "--tag", + "3.13.3", + "--owner", + "IBM", + "--repo", + "python-versions-pz", + "--assets-file", + str(assets_file), + ] + monkeypatch.setattr(sys, "argv", args) + + exit_code = gpm.main() + assert exit_code == 0 + + captured = capsys.readouterr() + payload = json.loads(captured.out) + assert payload[0]["download_url"].endswith("python-3.13.3-linux-22.04-ppc64le.tar.gz") + + +def test_main_rejects_invalid_urls(monkeypatch, capsys): + """Test that main() rejects URLs from wrong owner and returns error code.""" + assets = [ + { + "name": "python-3.13.3-linux-22.04-ppc64le.tar.gz", + "browser_download_url": "https://github.com/WRONG-OWNER/python-versions-pz/releases/download/3.13.3/python-3.13.3-linux-22.04-ppc64le.tar.gz", + } + ] + args = [ + "generate_partial_manifest.py", + "--tag", + "3.13.3", + "--owner", + "IBM", + "--repo", + "python-versions-pz", + "--assets", + json.dumps(assets), + ] + monkeypatch.setattr(sys, "argv", args) + + exit_code = gpm.main() + assert exit_code == 1 + + captured = capsys.readouterr() + assert "ERROR: No valid assets found after validation" in captured.err + + +def test_main_invalid_json(monkeypatch, capsys): + args = [ + "generate_partial_manifest.py", + "--tag", + "3.13.3", + "--owner", + "IBM", + "--repo", + "python-versions-pz", + "--assets", + "{invalid-json}", + ] + monkeypatch.setattr(sys, "argv", args) + + exit_code = gpm.main() + assert exit_code == 1 + + captured = capsys.readouterr() + assert "Error decoding JSON" in captured.err