feat: enhance manifest generation to support assets from file input

Signed-off-by: Adilhusain Shaikh <[email protected]>

Author: adilhusain-s

.github/scripts/generate_partial_manifest.py

@@ -111,13 +111,30 @@ def main() -> int:
     parser.add_argument("--tag", required=True, help="Release tag used for the manifest version field.")
     parser.add_argument("--owner", required=True, help="GitHub repository owner (e.g., 'IBM').")
     parser.add_argument("--repo", required=True, help="GitHub repository name (e.g., 'python-versions-pz').")
-    parser.add_argument("--assets", required=True, help="JSON string describing release assets from GitHub API.")
+    
+    # Mutually exclusive group: Accept EITHER string OR file
+    group = parser.add_mutually_exclusive_group(required=True)
+    group.add_argument("--assets", help="Raw JSON string (Legacy/Direct Input)")
+    group.add_argument("--assets-file", help="Path to JSON file containing assets (Recommended for CI)")
+    
     args = parser.parse_args()
 
+    # Logic to load assets from either source
+    assets = []
     try:
-        assets = json.loads(args.assets)
+        if args.assets_file:
+            print(f"Reading assets from file: {args.assets_file}", file=sys.stderr)
+            with open(args.assets_file, 'r', encoding='utf-8') as f:
+                assets = json.load(f)
+        elif args.assets:
+            # Legacy support for workflows passing raw strings
+            print(f"Parsing assets from command-line string", file=sys.stderr)
+            assets = json.loads(args.assets)
     except json.JSONDecodeError as exc:
-        print(f"Invalid JSON for --assets: {exc}", file=sys.stderr)
+        print(f"Error decoding JSON: {exc}", file=sys.stderr)
+        return 1
+    except FileNotFoundError as exc:
+        print(f"Assets file not found: {args.assets_file}", file=sys.stderr)
         return 1
 
     manifest_entries, errors = build_manifest_entries(args.tag, assets, args.owner, args.repo)

.github/workflows/backfill-manifests.yml

@@ -0,0 +1,117 @@
+name: Backfill Manifests
+
+on:
+  workflow_dispatch:
+    inputs:
+      limit:
+        description: "Number of releases to process (Max 250 due to GHA matrix limits)"
+        required: false
+        default: "30"
+        type: string
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  get-all-tags:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - name: Fetch release tags
+        id: set-matrix
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          LIMIT: ${{ inputs.limit }}
+        run: |
+          # Safety Check: Matrix strategy has a hard limit of 256 jobs
+          if [ "$LIMIT" -gt 250 ]; then
+            echo "::warning::Limit $LIMIT exceeds GitHub Matrix safety. Capping at 250."
+            LIMIT=250
+          fi
+
+          echo "Fetching last $LIMIT releases..."
+          
+          TAGS=$(gh release list \
+            --limit "$LIMIT" \
+            --exclude-drafts \
+            --exclude-pre-releases \
+            --json tagName \
+            --jq '[.[] | .tagName]')
+          
+          if [ -z "$TAGS" ] || [ "$TAGS" == "[]" ]; then
+            echo "::error::No release tags found!"
+            exit 1
+          fi
+
+          echo "Found tags count: $(echo $TAGS | jq '. | length')"
+          echo "matrix=$TAGS" >> $GITHUB_OUTPUT
+
+  generate-manifests:
+    needs: get-all-tags
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      max-parallel: 10 # Slightly higher parallelization is fine for reads
+      matrix:
+        tag: ${{ fromJson(needs.get-all-tags.outputs.matrix) }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.13
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+          cache: "poetry"
+
+      - name: Install dependencies
+        run: |
+          pip install poetry
+          poetry install --no-interaction --no-ansi
+
+      - name: Fetch Assets Data
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          echo "::group::Fetch Assets via GitHub CLI"
+          # Write JSON directly to file to bypass shell ARG_MAX limits
+          gh release view "${{ matrix.tag }}" --json assets --jq '.assets' > assets.json
+          
+          if [ ! -s assets.json ]; then
+            echo "::error::Assets file is empty for tag ${{ matrix.tag }}"
+            exit 1
+          fi
+          echo "::endgroup::"
+
+      - name: Generate Manifest
+        env:
+          REPO_NAME: ${{ github.repository }}
+          REPO_NAME: ${{ env.REPO_NAME }}
+          OWNER: ${{ github.repository_owner }}
+        run: |
+          echo "::group::Run Generation Script"
+          
+          REPO_NAME="${{ github.repository }}" && REPO_NAME="${REPO_NAME##*/}"
+          
+          # We use --assets-file now instead of --assets
+          poetry run python .github/scripts/generate_partial_manifest.py \
+            --tag "${{ matrix.tag }}" \
+            --owner "${{ github.repository_owner }}" \
+            --repo "${REPO_NAME}" \
+            --assets-file "assets.json" \
+            > manifest-part-${{ matrix.tag }}.json
+
+          cat manifest-part-${{ matrix.tag }}.json
+          echo "::endgroup::"
+
+      - name: Upload Manifest Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: manifest-part-${{ matrix.tag }}
+          path: manifest-part-${{ matrix.tag }}.json
+          retention-days: 1

tests/test_generate_partial_manifest.py

@@ -122,6 +122,39 @@ def test_main_outputs_manifest(monkeypatch, capsys):
     assert payload[0]["download_url"].endswith("python-3.13.3-linux-22.04-ppc64le.tar.gz")
 
 
+def test_main_outputs_manifest_from_file(monkeypatch, capsys, tmp_path):
+    """Test that main() reads from --assets-file correctly."""
+    assets = [
+        {
+            "name": "python-3.13.3-linux-22.04-ppc64le.tar.gz",
+            "browser_download_url": "https://github.com/IBM/python-versions-pz/releases/download/3.13.3/python-3.13.3-linux-22.04-ppc64le.tar.gz",
+        }
+    ]
+    
+    assets_file = tmp_path / "assets.json"
+    assets_file.write_text(json.dumps(assets), encoding='utf-8')
+    
+    args = [
+        "generate_partial_manifest.py",
+        "--tag",
+        "3.13.3",
+        "--owner",
+        "IBM",
+        "--repo",
+        "python-versions-pz",
+        "--assets-file",
+        str(assets_file),
+    ]
+    monkeypatch.setattr(sys, "argv", args)
+
+    exit_code = gpm.main()
+    assert exit_code == 0
+
+    captured = capsys.readouterr()
+    payload = json.loads(captured.out)
+    assert payload[0]["download_url"].endswith("python-3.13.3-linux-22.04-ppc64le.tar.gz")
+
+
 def test_main_rejects_invalid_urls(monkeypatch, capsys):
     """Test that main() rejects invalid URLs and returns error code."""
     assets = [