Add reusable ci workflow

.github/workflows/docs-ci.yml

@@ -0,0 +1,69 @@
+---
+name: "Docs CI"
+
+on:
+  workflow_call:
+    inputs:
+      pulpdocs_ref:
+        description: "Git reference of this workflow file. Must match the one used for the workflow call"
+        type: string
+        required: true
+
+jobs:
+  build:
+    runs-on: "ubuntu-latest"
+    steps:
+      - name: "Checkout pulp-docs repository"
+        uses: "actions/checkout@v4"
+        with:
+          repository: "pulp/pulp-docs"
+          path: "pulp-docs"
+          ref: "${{ inputs.pulpdocs_ref }}"
+          fetch-depth: 0
+
+      - name: "Checkout component repository"
+        if: "${{ github.event.repository.name != 'pulp-docs' }}"
+        uses: "actions/checkout@v4"
+        with:
+          path: "${{ github.event.repository.name }}"
+
+      - name: "Instructions to run locally"
+        run: |
+          cat <<EOL
+          Ensure you have "pulp-docs" and "${{ github.event.repository.name }}" in your CWD with correct revisions:
+
+          git -C pulp-docs checkout "${{ inputs.pulpdocs_ref }}"
+          git -C ${{ github.event.repository.name }} checkout "${{ github.head_ref }}"
+
+          Then run:
+
+          cd pulp-docs
+          pulp-docs fetch --dest /tmp/pulp-docs-tmp
+          pulp-docs build --path pulp-docs@..:${{ github.event.repository.name }}@..:/tmp/pulp-docs-tmp
+          EOL
+
+      - uses: "actions/setup-python@v5"
+        with:
+          python-version: "3.12"
+
+      - name: "Install pulp-docs"
+        working-directory: "pulp-docs"
+        run: |
+          pip install --upgrade pip
+          pip install -r doc_requirements.txt
+
+      - name: "Build Docs"
+        working-directory: "pulp-docs"
+        env:
+          DOCS_PATH: "pulp-docs@..:${{ github.event.repository.name }}@.."
+          DOCS_TMPDIR: "/tmp/pulp-docs-tmp"
+        run: |
+          pulp-docs fetch --path-exclude "$DOCS_PATH" --dest "$DOCS_TMPDIR"
+          pulp-docs build --path "$DOCS_PATH:$DOCS_TMPDIR"
+
+      - name: "Sanity Check"
+        working-directory: "pulp-docs"
+        run: |
+          echo "Checking that the namespace for the component under CI exists in the built docs."
+          ls "site/${{ github.event.repository.name }}/docs"
+...

.github/workflows/pr.yml

@@ -1,3 +1,4 @@
+---
 name: "pulp-docs CI"
 
 on:
@@ -9,7 +10,9 @@ concurrency:
 
 jobs:
   docs:
-    uses: "./.github/workflows/docs.yml"
+    uses: "./.github/workflows/docs-ci.yml"
+    with:
+      pulpdocs_ref: "${{ github.head_ref }}"
 
   tests:
     uses: "./.github/workflows/tests.yml"
@@ -27,3 +30,4 @@ jobs:
           echo '${{toJson(needs)}}' | jq -r 'to_entries[]|select(.value.result!="success")|.key + ": " + .value.result'
           echo '${{toJson(needs)}}' | jq -e 'to_entries|map(select(.value.result!="success"))|length == 0'
           echo "CI says: Looks good!"
+...

.github/workflows/tests.yml

@@ -1,3 +1,4 @@
+---
 name: "Tests"
 
 on:
@@ -20,3 +21,4 @@ jobs:
       - name: "Run test suite"
         run: |
           pytest -sv
+...

.github/workflows/update-data.yml

@@ -1,3 +1,4 @@
+---
 name: "Update Data Branch"
 
 on:
@@ -9,12 +10,12 @@ jobs:
   update-data:
     runs-on: "ubuntu-latest"
     permissions:
-      contents: write
+      contents: "write"
     steps:
       - name: "Checkout docs-data branch"
         uses: "actions/checkout@v4"
         with:
-          ref: docs-data
+          ref: "docs-data"
 
       - name: "Set up Python"
         uses: "actions/setup-python@v5"
@@ -40,3 +41,4 @@ jobs:
           git config user.name github-actions
           git config user.email [email protected]
           ./update-data.sh
+...

mkdocs.yml

@@ -135,6 +135,10 @@ plugins:
           git_url: "https://github.com/pulp/pulpcore"
           kind: "Other"
           # rest_api: "certguard"
+        - title: "Selinux"
+          path: "pulpcore-selinux"
+          git_url: "https://github.com/pulp/pulpcore-selinux"
+          kind: "Other"
 
         - title: "Pulp OCI Images"
           path: "pulp-oci-images"

src/pulp_docs/cli.py

@@ -6,6 +6,7 @@
 from mkdocs.__main__ import cli as mkdocs_cli
 from mkdocs.config import load_config
 from pulp_docs.context import ctx_blog, ctx_docstrings, ctx_draft, ctx_path
+from pulp_docs.plugin import load_components
 
 
 def blog_callback(ctx: click.Context, param: click.Parameter, value: bool) -> bool:
@@ -63,7 +64,7 @@ def find_path_callback(ctx: click.Context, param: click.Parameter, value: bool)
 )
 
 
-async def clone_repositories(repositories: list[str], dest_dir: Path) -> None:
+async def clone_repositories(repositories: set[str], dest_dir: Path) -> None:
     """Clone multiple repositories concurrently."""
 
     async def clone_repository(repo_url: str) -> None:
@@ -100,16 +101,25 @@ async def clone_repository(repo_url: str) -> None:
     envvar="PULPDOCS_DIR",
     help="Path to mkdocs.yml config file",
 )
-def fetch(dest, config_file):
+@click.option(
+    "--path-exclude",
+    default="",
+    callback=find_path_callback,
+    help="A colon separated list of lookup paths to exclude in the form [repo1@]path1 [:[repo2@]path2 [...]].",
+)
+def fetch(dest, config_file, path_exclude):
     """Fetch repositories to destination dir."""
     dest_path = Path(dest)
-    config = load_config(config_file)
-    components = config.plugins["PulpDocs"].config.components
-    repositories_list = list({r.git_url for r in components})
+    pulpdocs_plugin = load_config(config_file).plugins["PulpDocs"]
+    all_components = pulpdocs_plugin.config.components
+    all_repositories_set = {r.git_url for r in all_components if r.git_url}
+    found_components = load_components(path_exclude, pulpdocs_plugin.config, draft=True)
+    found_repositories_set = {r.git_url for r in found_components}
+    final_repositories_set = all_repositories_set - found_repositories_set
 
     if not dest_path.exists():
         dest_path.mkdir(parents=True)
-    asyncio.run(clone_repositories(repositories_list, dest_path))
+    asyncio.run(clone_repositories(final_repositories_set, dest_path))
 
 
 main = mkdocs_cli

src/pulp_docs/plugin.py

@@ -257,7 +257,7 @@ def rss_items() -> list:
     return rss_feed["items"][:20]
 
 
-def load_components(find_path: list[str], config: MkDocsConfig, draft: bool):
+def load_components(find_path: list[str], config: PulpDocsPluginConfig, draft: bool):
     loaded_components = []
     for component_opt in config.components:
         component = Component.build(find_path, component_opt)