Parallelize Antora component build

.github/workflows/ci.yml

@@ -113,10 +113,21 @@ jobs:
         with:
           node-version: '20.12.2'
           cache: 'npm'
-          cache-dependency-path: docs-site/package-lock.json
+          cache-dependency-path: |
+            docs-site/package-lock.json
+            antora-ui-khronos/package-lock.json
+            Vulkan-Docs/package-lock.json
 
-      - name: "force clear the npm cache"
-        run: npm cache clean --force
+      - name: "Prepare Antora cache dir"
+        run: mkdir -p docs-site/build/.cache
+
+      - name: "Cache Antora workdir"
+        uses: actions/cache@v4
+        with:
+          path: docs-site/build/.cache
+          key: antora-cache-${{ hashFiles('docs-site/antora-playbook.yml', 'docs-site/package-lock.json') }}
+          restore-keys: |
+            antora-cache-
 
       - name: "run npm install for ui bundle"
         working-directory: antora-ui-khronos
@@ -167,10 +178,14 @@ jobs:
         working-directory: Vulkan-Samples
         run: cmake -H"." -B"build/unix" -DVKB_GENERATE_ANTORA_SITE=ON
 
-      - name: "build (npx) with stacktrace"
+      - name: "build site in parallel (fan-out with Lunr)"
         working-directory: docs-site
+        env:
+          ANTORA_LUNR_NODE_MAX_OLD_SPACE: '6144'
+          ANTORA_LUNR_IO_WORKERS: '1'
+          ANTORA_LUNR_SIMPLE_PIPELINE: '1'
         run: |
-          npx antora antora-playbook.yml --stacktrace
+          npm run antora-fanout
           touch build/site/.nojekyll
 
       - name: 'Upload site artifact'

Makefile

@@ -79,22 +79,22 @@ prep-docs:
 	   docs-site/js/
 
 prep-glsl:
-	make -C GLSL clean setup_antora
+	$(MAKE) -C GLSL clean setup_antora
 
 prep-guide:
-	make -C Vulkan-Guide -f antora/Makefile clean setup
+	$(MAKE) -C Vulkan-Guide -f antora/Makefile clean setup
 
 prep-samples:
 	cd Vulkan-Samples && cmake -H"." -B"build/unix" -DVKB_GENERATE_ANTORA_SITE=ON
 
 prep-tutorial:
-	make -C Vulkan-Tutorial/antora setup_tutorial
+	$(MAKE) -C Vulkan-Tutorial/antora setup_tutorial
 
 # Build Antora site
 # CI is needed as an environment variable which helps cause suppression
 # of the "Edit this Page" link otherwise generated.
 export CI = true
-build-site:
+build-site: build-ui prep-sources
 	cd docs-site && npx antora antora-playbook.yml --stacktrace
 
 # Clean Antora site (but not prepared component sources)

docs-site/antora-playbook.yml

@@ -31,6 +31,9 @@ antora:
   extensions:
     - require: '@antora/lunr-extension'
       index_latest_only: true
+    - require: antora-sources-parallel
+      sources_parallel: true
+      experimental_fanout: true
 asciidoc:
   extensions:
     # specmacros.js requires './apimap.cjs', 'xrefMap.cjs', and 'pageMap.cjs'.

docs-site/extensions/antora-sources-parallel/README.md

@@ -0,0 +1,92 @@
+# antora-sources-parallel
+
+Antora extension to enable parallel-friendly builds. Phase 1 provides safe concurrency hints; a future phase can implement per-source fan-out.
+
+## Install / Use (local)
+
+In your Antora playbook:
+
+```yaml
+antora:
+  extensions:
+    - require: ./extensions/antora-sources-parallel
+      sources_parallel: true
+      # Optional tuning:
+      # min_workers: 3
+      # max_workers: 8
+```
+
+This extension computes a worker count based on your CPU core count (or `min_workers`, default 3), and sets the following environment variables if not already set:
+- `ANTORA_FETCH_CONCURRENCY`
+- `ANTORA_CONCURRENCY`
+- `ANTORA_SOURCES_PARALLEL_WORKERS`
+
+These hints can be used by Antora and cooperating extensions to perform work in parallel.
+
+## As a separate package
+
+When extracted to its own repository and published, you can use:
+
+```yaml
+antora:
+  extensions:
+    - require: antora-sources-parallel
+      sources_parallel: true
+```
+
+## Experimental features
+
+- `experimental_fanout: true` is reserved for a future release supporting per-source fan-out. It is currently not implemented and ignored aside from a warning.
+
+## License
+
+Apache-2.0
+
+
+## Per-source fan-out (experimental)
+
+This package includes an optional fan-out orchestrator that builds each content source in parallel and merges outputs.
+
+How to use locally:
+
+- Ensure you are in the docs-site directory and have installed dependencies (npm install)
+- Run:
+
+```
+npm run antora-fanout
+```
+
+What it does:
+- Splits the playbook’s content.sources into separate temporary playbooks
+- Runs `npx antora` for each in parallel (workers derived from CPU count or env), outputting to build/.fanout/<idx>
+- Merges all shard outputs into build/site
+
+Tuning:
+- Set one of these env vars to control concurrency: ANTORA_SOURCES_PARALLEL_WORKERS, ANTORA_CONCURRENCY, ANTORA_FETCH_CONCURRENCY
+
+Notes:
+- The standard build (npx antora antora-playbook.yml) remains unchanged; fan-out is opt-in
+- Collisions in generated files are resolved "last writer wins" during merge
+
+## CI usage and performance tips
+
+- Fast path in CI: run the parallel fan-out without Lunr indexing.
+  - From docs-site/: npm run antora:ci
+  - Equivalent: node ./extensions/antora-sources-parallel/bin/fanout.js antora-playbook.yml --no-lunr
+  - You can also set ANTORA_NO_LUNR=1 instead of passing --no-lunr.
+
+- Concurrency tuning:
+  - ANTORA_SOURCES_PARALLEL_WORKERS: hard limit for shard workers.
+  - ANTORA_CONCURRENCY / ANTORA_FETCH_CONCURRENCY: generic hints also used by some tools.
+  - Default worker count is max(cpu cores, 3).
+
+- Caching:
+  - The extension sets ANTORA_CACHE_DIR to build/.cache if not already set.
+  - Configure your CI to cache the docs-site/build/.cache directory between runs to reduce repeated work.
+
+- Release builds (with search index):
+  - Use npm run antora-fanout to keep Lunr enabled while still running per-source in parallel.
+  - Or run the standard Antora command if you prefer the traditional single-process path.
+
+- No Makefile changes required:
+  - Keep Makefile as-is; point CI to run the npm script from docs-site instead.

docs-site/extensions/antora-sources-parallel/bin/build-lunr-from-site.js

@@ -0,0 +1,183 @@
+#!/usr/bin/env node
+/*
+ * Fast Lunr indexer: builds a global search index directly from the merged Antora site HTML.
+ *
+ * Usage:
+ *   node build-lunr-from-site.js <siteDir>
+ *
+ * Output:
+ *   <siteDir>/search-index.json
+ *   <siteDir>/search-index.js  (assigns window.searchIndex = { ... })
+ */
+
+const fs = require('fs')
+const fsp = fs.promises
+const path = require('path')
+const os = require('os')
+const lunr = require('lunr')
+
+function cpuWorkers (min = 3) {
+  const cores = Array.isArray(os.cpus()) ? os.cpus().length : 1
+  return Math.max(min, cores || 1)
+}
+
+function getWorkers () {
+  const envKeys = ['ANTORA_LUNR_IO_WORKERS', 'ANTORA_SOURCES_PARALLEL_WORKERS', 'ANTORA_CONCURRENCY', 'ANTORA_FETCH_CONCURRENCY']
+  for (const k of envKeys) {
+    const v = process.env[k]
+    if (v && +v > 0) return +v
+  }
+  // Default to ultra-conservative IO concurrency to minimize memory pressure
+  return 1
+}
+
+function getMaxCharsPerPage () {
+  const v = process.env.ANTORA_LUNR_MAX_CHARS
+  if (v && +v > 0) return +v
+  // Tighter upper bound to avoid pathological pages blowing memory
+  return 60000
+}
+
+function stripHtml (html) {
+  // Remove script/style contents
+  html = html.replace(/<script[\s\S]*?<\/script>/gi, ' ').replace(/<style[\s\S]*?<\/style>/gi, ' ')
+  // Replace tags with spaces
+  html = html.replace(/<[^>]+>/g, ' ')
+  // Decode a few common entities
+  html = html.replace(/&nbsp;/g, ' ').replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>')
+  // Collapse whitespace
+  return html.replace(/\s+/g, ' ').trim()
+}
+
+async function readFileSafe (file) {
+  try {
+    return await fsp.readFile(file, 'utf8')
+  } catch (e) {
+    return ''
+  }
+}
+
+function extractTitle (html, fallback) {
+  const h1 = html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i)
+  if (h1 && h1[1]) return stripHtml(h1[1])
+  const title = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)
+  if (title && title[1]) return stripHtml(title[1])
+  return fallback || ''
+}
+
+async function listHtmlFiles (dir) {
+  const out = []
+  async function walk (d) {
+    const entries = await fsp.readdir(d, { withFileTypes: true })
+    for (const ent of entries) {
+      const p = path.join(d, ent.name)
+      if (ent.isDirectory()) {
+        // skip some known non-page dirs if present
+        if (ent.name === '_') continue
+        await walk(p)
+      } else if (ent.isFile()) {
+        if (p.endsWith('.html')) out.push(p)
+      }
+    }
+  }
+  await walk(dir)
+  return out
+}
+
+function toSiteUrl (siteDir, filePath) {
+  // Convert absolute file path back to site-relative URL
+  const rel = path.relative(siteDir, filePath)
+  let url = rel.replace(/\\/g, '/')
+  // Ensure it starts with a slash for UI expectations
+  if (!url.startsWith('/')) url = '/' + url
+  return url
+}
+
+async function buildIndex (siteDir) {
+  let files = await listHtmlFiles(siteDir)
+  // Index latest-only by default if such pages exist
+  const hasLatest = files.some((f) => f.includes(`${path.sep}latest${path.sep}`) || f.includes('/latest/'))
+  if (hasLatest) {
+    files = files.filter((f) => f.includes(`${path.sep}latest${path.sep}`) || f.includes('/latest/'))
+  }
+  const ioWorkers = getWorkers()
+  const maxChars = getMaxCharsPerPage()
+
+  // Minimal doc metadata to ship with the index
+  const docMeta = []
+
+  // Prepare a Lunr builder so we can add docs incrementally (low memory)
+  const builder = new lunr.Builder()
+  builder.ref('id')
+  builder.field('title', { boost: 10 })
+  builder.field('text')
+
+  // Optional simplified pipeline to reduce memory/CPU
+  const simplePipeline = (process.env.ANTORA_LUNR_SIMPLE_PIPELINE || '1') === '1'
+  if (simplePipeline) {
+    // Remove heavy stemming/stopword filters; keep minimal trimmer
+    builder.pipeline.reset()
+    builder.searchPipeline.reset()
+    if (lunr.trimmer) {
+      builder.pipeline.add(lunr.trimmer)
+      builder.searchPipeline.add(lunr.trimmer)
+    }
+  }
+
+  // Process files in small concurrent batches for IO, but add to index immediately
+  let idxNext = 0
+  let inFlight = 0
+  await new Promise((resolve, reject) => {
+    const next = () => {
+      while (inFlight < ioWorkers && idxNext < files.length) {
+        const file = files[idxNext++]
+        inFlight++
+        ;(async () => {
+          const html = await readFileSafe(file)
+          const url = toSiteUrl(siteDir, file)
+          const title = extractTitle(html, path.basename(file, '.html'))
+          let text = stripHtml(html)
+          if (maxChars && text.length > maxChars) text = text.slice(0, maxChars)
+          // Add to index and discard text immediately
+          builder.add({ id: url, title, text })
+          docMeta.push({ id: url, title, url })
+        })()
+          .then(() => {
+            inFlight--
+            if (idxNext >= files.length && inFlight === 0) resolve()
+            else next()
+          })
+          .catch((e) => reject(e))
+      }
+      if (idxNext >= files.length && inFlight === 0) resolve()
+    }
+    next()
+  })
+
+  const index = builder.build()
+  return { docs: docMeta, index: index.toJSON() }
+}
+
+async function writeOutputs (siteDir, payload) {
+  const jsonPath = path.join(siteDir, 'search-index.json')
+  const jsPath = path.join(siteDir, 'search-index.js')
+  const json = JSON.stringify(payload)
+  await fsp.writeFile(jsonPath, json, 'utf8')
+  const js = `window.searchIndex=${json};\n`
+  await fsp.writeFile(jsPath, js, 'utf8')
+}
+
+async function main () {
+  const siteDir = path.resolve(process.argv[2] || path.join(process.cwd(), 'build', 'site'))
+  console.log(`[lunr-fast] Indexing site at: ${siteDir}`)
+  const t0 = Date.now()
+  const payload = await buildIndex(siteDir)
+  await writeOutputs(siteDir, payload)
+  const dt = ((Date.now() - t0) / 1000).toFixed(2)
+  console.log(`[lunr-fast] Wrote search-index.json and search-index.js in ${dt}s`)
+}
+
+main().catch((e) => {
+  console.error('[lunr-fast] fatal:', e && e.message ? e.message : e)
+  process.exit(1)
+})