diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da92674e53..d8809914a8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -113,10 +113,21 @@ jobs: with: node-version: '20.12.2' cache: 'npm' - cache-dependency-path: docs-site/package-lock.json + cache-dependency-path: | + docs-site/package-lock.json + antora-ui-khronos/package-lock.json + Vulkan-Docs/package-lock.json - - name: "force clear the npm cache" - run: npm cache clean --force + - name: "Prepare Antora cache dir" + run: mkdir -p docs-site/build/.cache + + - name: "Cache Antora workdir" + uses: actions/cache@v4 + with: + path: docs-site/build/.cache + key: antora-cache-${{ hashFiles('docs-site/antora-playbook.yml', 'docs-site/package-lock.json') }} + restore-keys: | + antora-cache- - name: "run npm install for ui bundle" working-directory: antora-ui-khronos @@ -167,10 +178,14 @@ jobs: working-directory: Vulkan-Samples run: cmake -H"." -B"build/unix" -DVKB_GENERATE_ANTORA_SITE=ON - - name: "build (npx) with stacktrace" + - name: "build site in parallel (fan-out with Lunr)" working-directory: docs-site + env: + ANTORA_LUNR_NODE_MAX_OLD_SPACE: '6144' + ANTORA_LUNR_IO_WORKERS: '1' + ANTORA_LUNR_SIMPLE_PIPELINE: '1' run: | - npx antora antora-playbook.yml --stacktrace + npm run antora-fanout touch build/site/.nojekyll - name: 'Upload site artifact' diff --git a/Makefile b/Makefile index 532505c3e8..66e7562dcf 100644 --- a/Makefile +++ b/Makefile @@ -79,22 +79,22 @@ prep-docs: docs-site/js/ prep-glsl: - make -C GLSL clean setup_antora + $(MAKE) -C GLSL clean setup_antora prep-guide: - make -C Vulkan-Guide -f antora/Makefile clean setup + $(MAKE) -C Vulkan-Guide -f antora/Makefile clean setup prep-samples: cd Vulkan-Samples && cmake -H"." -B"build/unix" -DVKB_GENERATE_ANTORA_SITE=ON prep-tutorial: - make -C Vulkan-Tutorial/antora setup_tutorial + $(MAKE) -C Vulkan-Tutorial/antora setup_tutorial # Build Antora site # CI is needed as an environment variable which helps cause suppression # of the "Edit this Page" link otherwise generated. export CI = true -build-site: +build-site: build-ui prep-sources cd docs-site && npx antora antora-playbook.yml --stacktrace # Clean Antora site (but not prepared component sources) diff --git a/antora-ui-khronos b/antora-ui-khronos index ac0b12aadf..67fb0f7cb9 160000 --- a/antora-ui-khronos +++ b/antora-ui-khronos @@ -1 +1 @@ -Subproject commit ac0b12aadf11db0625445dd5c3eaf374f1176df8 +Subproject commit 67fb0f7cb94ee9993dd3f1dac86d6295016c4901 diff --git a/docs-site/antora-playbook.yml b/docs-site/antora-playbook.yml index f007488b49..d14f0aa2da 100644 --- a/docs-site/antora-playbook.yml +++ b/docs-site/antora-playbook.yml @@ -31,6 +31,9 @@ antora: extensions: - require: '@antora/lunr-extension' index_latest_only: true + - require: antora-sources-parallel + sources_parallel: true + experimental_fanout: true asciidoc: extensions: # specmacros.js requires './apimap.cjs', 'xrefMap.cjs', and 'pageMap.cjs'. diff --git a/docs-site/extensions/antora-sources-parallel/README.md b/docs-site/extensions/antora-sources-parallel/README.md new file mode 100644 index 0000000000..115bfc6394 --- /dev/null +++ b/docs-site/extensions/antora-sources-parallel/README.md @@ -0,0 +1,92 @@ +# antora-sources-parallel + +Antora extension to enable parallel-friendly builds. Phase 1 provides safe concurrency hints; a future phase can implement per-source fan-out. + +## Install / Use (local) + +In your Antora playbook: + +```yaml +antora: + extensions: + - require: ./extensions/antora-sources-parallel + sources_parallel: true + # Optional tuning: + # min_workers: 3 + # max_workers: 8 +``` + +This extension computes a worker count based on your CPU core count (or `min_workers`, default 3), and sets the following environment variables if not already set: +- `ANTORA_FETCH_CONCURRENCY` +- `ANTORA_CONCURRENCY` +- `ANTORA_SOURCES_PARALLEL_WORKERS` + +These hints can be used by Antora and cooperating extensions to perform work in parallel. + +## As a separate package + +When extracted to its own repository and published, you can use: + +```yaml +antora: + extensions: + - require: antora-sources-parallel + sources_parallel: true +``` + +## Experimental features + +- `experimental_fanout: true` is reserved for a future release supporting per-source fan-out. It is currently not implemented and ignored aside from a warning. + +## License + +Apache-2.0 + + +## Per-source fan-out (experimental) + +This package includes an optional fan-out orchestrator that builds each content source in parallel and merges outputs. + +How to use locally: + +- Ensure you are in the docs-site directory and have installed dependencies (npm install) +- Run: + +``` +npm run antora-fanout +``` + +What it does: +- Splits the playbook’s content.sources into separate temporary playbooks +- Runs `npx antora` for each in parallel (workers derived from CPU count or env), outputting to build/.fanout/ +- Merges all shard outputs into build/site + +Tuning: +- Set one of these env vars to control concurrency: ANTORA_SOURCES_PARALLEL_WORKERS, ANTORA_CONCURRENCY, ANTORA_FETCH_CONCURRENCY + +Notes: +- The standard build (npx antora antora-playbook.yml) remains unchanged; fan-out is opt-in +- Collisions in generated files are resolved "last writer wins" during merge + +## CI usage and performance tips + +- Fast path in CI: run the parallel fan-out without Lunr indexing. + - From docs-site/: npm run antora:ci + - Equivalent: node ./extensions/antora-sources-parallel/bin/fanout.js antora-playbook.yml --no-lunr + - You can also set ANTORA_NO_LUNR=1 instead of passing --no-lunr. + +- Concurrency tuning: + - ANTORA_SOURCES_PARALLEL_WORKERS: hard limit for shard workers. + - ANTORA_CONCURRENCY / ANTORA_FETCH_CONCURRENCY: generic hints also used by some tools. + - Default worker count is max(cpu cores, 3). + +- Caching: + - The extension sets ANTORA_CACHE_DIR to build/.cache if not already set. + - Configure your CI to cache the docs-site/build/.cache directory between runs to reduce repeated work. + +- Release builds (with search index): + - Use npm run antora-fanout to keep Lunr enabled while still running per-source in parallel. + - Or run the standard Antora command if you prefer the traditional single-process path. + +- No Makefile changes required: + - Keep Makefile as-is; point CI to run the npm script from docs-site instead. diff --git a/docs-site/extensions/antora-sources-parallel/bin/build-lunr-from-site.js b/docs-site/extensions/antora-sources-parallel/bin/build-lunr-from-site.js new file mode 100644 index 0000000000..8faf94c2c8 --- /dev/null +++ b/docs-site/extensions/antora-sources-parallel/bin/build-lunr-from-site.js @@ -0,0 +1,183 @@ +#!/usr/bin/env node +/* + * Fast Lunr indexer: builds a global search index directly from the merged Antora site HTML. + * + * Usage: + * node build-lunr-from-site.js + * + * Output: + * /search-index.json + * /search-index.js (assigns window.searchIndex = { ... }) + */ + +const fs = require('fs') +const fsp = fs.promises +const path = require('path') +const os = require('os') +const lunr = require('lunr') + +function cpuWorkers (min = 3) { + const cores = Array.isArray(os.cpus()) ? os.cpus().length : 1 + return Math.max(min, cores || 1) +} + +function getWorkers () { + const envKeys = ['ANTORA_LUNR_IO_WORKERS', 'ANTORA_SOURCES_PARALLEL_WORKERS', 'ANTORA_CONCURRENCY', 'ANTORA_FETCH_CONCURRENCY'] + for (const k of envKeys) { + const v = process.env[k] + if (v && +v > 0) return +v + } + // Default to ultra-conservative IO concurrency to minimize memory pressure + return 1 +} + +function getMaxCharsPerPage () { + const v = process.env.ANTORA_LUNR_MAX_CHARS + if (v && +v > 0) return +v + // Tighter upper bound to avoid pathological pages blowing memory + return 60000 +} + +function stripHtml (html) { + // Remove script/style contents + html = html.replace(//gi, ' ').replace(//gi, ' ') + // Replace tags with spaces + html = html.replace(/<[^>]+>/g, ' ') + // Decode a few common entities + html = html.replace(/ /g, ' ').replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>') + // Collapse whitespace + return html.replace(/\s+/g, ' ').trim() +} + +async function readFileSafe (file) { + try { + return await fsp.readFile(file, 'utf8') + } catch (e) { + return '' + } +} + +function extractTitle (html, fallback) { + const h1 = html.match(/]*>([\s\S]*?)<\/h1>/i) + if (h1 && h1[1]) return stripHtml(h1[1]) + const title = html.match(/]*>([\s\S]*?)<\/title>/i) + if (title && title[1]) return stripHtml(title[1]) + return fallback || '' +} + +async function listHtmlFiles (dir) { + const out = [] + async function walk (d) { + const entries = await fsp.readdir(d, { withFileTypes: true }) + for (const ent of entries) { + const p = path.join(d, ent.name) + if (ent.isDirectory()) { + // skip some known non-page dirs if present + if (ent.name === '_') continue + await walk(p) + } else if (ent.isFile()) { + if (p.endsWith('.html')) out.push(p) + } + } + } + await walk(dir) + return out +} + +function toSiteUrl (siteDir, filePath) { + // Convert absolute file path back to site-relative URL + const rel = path.relative(siteDir, filePath) + let url = rel.replace(/\\/g, '/') + // Ensure it starts with a slash for UI expectations + if (!url.startsWith('/')) url = '/' + url + return url +} + +async function buildIndex (siteDir) { + let files = await listHtmlFiles(siteDir) + // Index latest-only by default if such pages exist + const hasLatest = files.some((f) => f.includes(`${path.sep}latest${path.sep}`) || f.includes('/latest/')) + if (hasLatest) { + files = files.filter((f) => f.includes(`${path.sep}latest${path.sep}`) || f.includes('/latest/')) + } + const ioWorkers = getWorkers() + const maxChars = getMaxCharsPerPage() + + // Minimal doc metadata to ship with the index + const docMeta = [] + + // Prepare a Lunr builder so we can add docs incrementally (low memory) + const builder = new lunr.Builder() + builder.ref('id') + builder.field('title', { boost: 10 }) + builder.field('text') + + // Optional simplified pipeline to reduce memory/CPU + const simplePipeline = (process.env.ANTORA_LUNR_SIMPLE_PIPELINE || '1') === '1' + if (simplePipeline) { + // Remove heavy stemming/stopword filters; keep minimal trimmer + builder.pipeline.reset() + builder.searchPipeline.reset() + if (lunr.trimmer) { + builder.pipeline.add(lunr.trimmer) + builder.searchPipeline.add(lunr.trimmer) + } + } + + // Process files in small concurrent batches for IO, but add to index immediately + let idxNext = 0 + let inFlight = 0 + await new Promise((resolve, reject) => { + const next = () => { + while (inFlight < ioWorkers && idxNext < files.length) { + const file = files[idxNext++] + inFlight++ + ;(async () => { + const html = await readFileSafe(file) + const url = toSiteUrl(siteDir, file) + const title = extractTitle(html, path.basename(file, '.html')) + let text = stripHtml(html) + if (maxChars && text.length > maxChars) text = text.slice(0, maxChars) + // Add to index and discard text immediately + builder.add({ id: url, title, text }) + docMeta.push({ id: url, title, url }) + })() + .then(() => { + inFlight-- + if (idxNext >= files.length && inFlight === 0) resolve() + else next() + }) + .catch((e) => reject(e)) + } + if (idxNext >= files.length && inFlight === 0) resolve() + } + next() + }) + + const index = builder.build() + return { docs: docMeta, index: index.toJSON() } +} + +async function writeOutputs (siteDir, payload) { + const jsonPath = path.join(siteDir, 'search-index.json') + const jsPath = path.join(siteDir, 'search-index.js') + const json = JSON.stringify(payload) + await fsp.writeFile(jsonPath, json, 'utf8') + const js = `window.searchIndex=${json};\n` + await fsp.writeFile(jsPath, js, 'utf8') +} + +async function main () { + const siteDir = path.resolve(process.argv[2] || path.join(process.cwd(), 'build', 'site')) + console.log(`[lunr-fast] Indexing site at: ${siteDir}`) + const t0 = Date.now() + const payload = await buildIndex(siteDir) + await writeOutputs(siteDir, payload) + const dt = ((Date.now() - t0) / 1000).toFixed(2) + console.log(`[lunr-fast] Wrote search-index.json and search-index.js in ${dt}s`) +} + +main().catch((e) => { + console.error('[lunr-fast] fatal:', e && e.message ? e.message : e) + process.exit(1) +}) diff --git a/docs-site/extensions/antora-sources-parallel/bin/fanout.js b/docs-site/extensions/antora-sources-parallel/bin/fanout.js new file mode 100644 index 0000000000..394becec1c --- /dev/null +++ b/docs-site/extensions/antora-sources-parallel/bin/fanout.js @@ -0,0 +1,352 @@ +#!/usr/bin/env node +/* + * antora-sources-parallel: per-source fan-out orchestrator (experimental) + * + * Usage (from docs-site/): + * node ./extensions/antora-sources-parallel/bin/fanout.js antora-playbook.yml + * + * Behavior: + * - Reads the provided playbook + * - Splits content.sources into one playbook per source + * - Runs `npx antora ` for each in parallel (bounded by worker limit) + * - Each run outputs into build/.fanout/ + * - After completion, merges all into build/site (last write wins on collisions) + * + * Notes: + * - This is an optional runner; it does not modify Antora itself. + * - Requires dev deps: @antora/cli, @antora/site-generator in docs-site + * - Uses js-yaml as a dependency of the local extension package. + */ + +const fs = require('fs') +const fsp = fs.promises +const path = require('path') +const { spawn } = require('child_process') +const yaml = require('js-yaml') +const os = require('os') + +function cpuWorkers (min = 3) { + const cores = Array.isArray(os.cpus()) ? os.cpus().length : 1 + return Math.max(min, cores || 1) +} + +function getWorkersFromEnv () { + const envKeys = ['ANTORA_SOURCES_PARALLEL_WORKERS', 'ANTORA_CONCURRENCY', 'ANTORA_FETCH_CONCURRENCY'] + for (const k of envKeys) { + const v = process.env[k] + if (v && +v > 0) return +v + } + return cpuWorkers(3) +} + +async function readYaml (file) { + const text = await fsp.readFile(file, 'utf8') + return yaml.load(text) +} + +async function writeYaml (file, obj) { + const text = yaml.dump(obj, { noRefs: true, lineWidth: 120 }) + await fsp.writeFile(file, text, 'utf8') +} + +async function rimraf (p) { + await fsp.rm(p, { recursive: true, force: true }) +} + +async function mkdirp (p) { + await fsp.mkdir(p, { recursive: true }) +} + +function runAntora (cwd, playbookPath) { + return new Promise((resolve, reject) => { + const cacheDir = process.env.ANTORA_CACHE_DIR + const args = ['antora', playbookPath, '--stacktrace'] + if (cacheDir) { + args.push('--cache-dir', cacheDir) + } + const child = spawn(process.platform === 'win32' ? 'npx.cmd' : 'npx', args, { + cwd, + stdio: 'inherit', + env: process.env, + }) + child.on('exit', (code) => { + if (code === 0) resolve() + else reject(new Error(`antora exited with code ${code}`)) + }) + child.on('error', reject) + }) +} + +async function copyRecursive (src, dest) { + const st = await fsp.stat(src) + if (st.isDirectory()) { + await mkdirp(dest) + const entries = await fsp.readdir(src) + for (const e of entries) { + await copyRecursive(path.join(src, e), path.join(dest, e)) + } + } else if (st.isFile()) { + await mkdirp(path.dirname(dest)) + await fsp.copyFile(src, dest) + } +} + +async function mergeDirs (srcDir, dstDir) { + await mkdirp(dstDir) + const entries = await fsp.readdir(srcDir) + for (const e of entries) { + await copyRecursive(path.join(srcDir, e), path.join(dstDir, e)) + } +} + +function isLikelyUrl (val) { + return typeof val === 'string' && /^(https?:)?\/\//i.test(val) +} + +function isRelativePath (val) { + return typeof val === 'string' && (val.startsWith('./') || val.startsWith('../')) +} + +function rebasePlaybookPaths (pb, rootDir) { + if (!pb || !rootDir) return pb + // Rebase AsciiDoc extensions that are relative paths + if (pb.asciidoc && Array.isArray(pb.asciidoc.extensions)) { + pb.asciidoc.extensions = pb.asciidoc.extensions.map((ext) => { + if (typeof ext === 'string' && isRelativePath(ext)) { + return path.resolve(rootDir, ext) + } else if (ext && typeof ext === 'object' && isRelativePath(ext.require)) { + return { ...ext, require: path.resolve(rootDir, ext.require) } + } + return ext + }) + } + // Rebase Antora extensions whose require is a relative path + if (pb.antora && Array.isArray(pb.antora.extensions)) { + pb.antora.extensions = pb.antora.extensions.map((e) => { + if (e && typeof e === 'object' && typeof e.require === 'string' && isRelativePath(e.require)) { + return { ...e, require: path.resolve(rootDir, e.require) } + } + return e + }) + } + // Rebase UI bundle local file URL + if (pb.ui && pb.ui.bundle && typeof pb.ui.bundle.url === 'string') { + const u = pb.ui.bundle.url + if (!isLikelyUrl(u) && !path.isAbsolute(u)) { + pb.ui.bundle.url = path.resolve(rootDir, u) + } + } + return pb +} + +async function main () { + const docsSiteDir = process.cwd() + const argv = process.argv.slice(2) + const playbookArg = argv[0] || 'antora-playbook.yml' + const disableLunr = argv.includes('--no-lunr') || process.env.ANTORA_NO_LUNR === '1' + const playbookPath = path.resolve(docsSiteDir, playbookArg) + + // Encourage cache reuse if not already configured + if (!process.env.ANTORA_CACHE_DIR) { + process.env.ANTORA_CACHE_DIR = path.join(docsSiteDir, 'build', '.cache') + } + await mkdirp(process.env.ANTORA_CACHE_DIR) + console.log(`[fanout] Using Antora cache dir: ${process.env.ANTORA_CACHE_DIR}`) + + const playbook = await readYaml(playbookPath) + const sources = playbook?.content?.sources + if (!Array.isArray(sources) || sources.length === 0) { + throw new Error('No content.sources found in playbook') + } + + // Expand any sources that specify start_paths into separate shards per path + const expandedSources = [] + for (const s of sources) { + if (s && Array.isArray(s.start_paths) && s.start_paths.length > 0) { + for (const sp of s.start_paths) { + const ns = { ...s } + delete ns.start_paths + ns.start_path = sp + expandedSources.push(ns) + } + } else { + expandedSources.push(s) + } + } + + const workers = getWorkersFromEnv() + const fanoutRoot = path.join(docsSiteDir, 'build', '.fanout') + const finalOut = path.join(docsSiteDir, 'build', 'site') + + await rimraf(fanoutRoot) + await mkdirp(fanoutRoot) + + // Prepare per-source playbooks + const tasks = expandedSources.map((src, idx) => ({ src, idx })) + + // Create temp playbooks with single source and per-run output dir + const tempPlaybooks = [] + for (const { src, idx } of tasks) { + let pb = JSON.parse(JSON.stringify(playbook)) + pb.content.sources = [src] + + // Remove lunr extension for shard builds to avoid per-shard indexing; we'll build it once globally later (unless disabled) + if (pb.antora && Array.isArray(pb.antora.extensions)) { + pb.antora.extensions = pb.antora.extensions.filter((e) => !(e && e.require === '@antora/lunr-extension')) + } + + // Rebase any relative paths to absolute paths from docs-site root so antora resolves them correctly + pb = rebasePlaybookPaths(pb, docsSiteDir) + + // Ensure output writes to unique dir per shard + pb.output = pb.output || {} + pb.output.dir = path.join('build', '.fanout', String(idx)) + + const file = path.join(fanoutRoot, `playbook-${idx}.yml`) + await writeYaml(file, pb) + tempPlaybooks.push({ idx, file, outDir: path.join(docsSiteDir, pb.output.dir) }) + } + + // Run in parallel with a simple pool + let inFlight = 0 + let i = 0 + let failed = false + + await rimraf(finalOut) + await mkdirp(finalOut) + + await new Promise((resolve) => { + const next = () => { + if (failed) return + if (i >= tempPlaybooks.length && inFlight === 0) return resolve() + while (inFlight < workers && i < tempPlaybooks.length) { + const { file, idx } = tempPlaybooks[i++] + inFlight++ + runAntora(docsSiteDir, file) + .then(() => { + inFlight-- + next() + }) + .catch((err) => { + console.error(`[fanout] build failed for shard ${idx}:`, err.message) + failed = true + inFlight-- + next() + }) + } + } + next() + }) + + if (failed) { + process.exitCode = 1 + console.error('[fanout] One or more shards failed') + return + } + + // Merge outputs + for (const { outDir } of tempPlaybooks) { + // Expect outDir like build/.fanout/ + await mergeDirs(outDir, finalOut) + } + + console.log(`[fanout] Completed per-source build for ${tasks.length} sources with workers=${workers}`) + console.log(`[fanout] Output merged to: ${finalOut}`) + + // If Lunr is not explicitly disabled, build a global search index + if (!disableLunr) { + const mode = (process.env.ANTORA_LUNR_MODE || 'fast').toLowerCase() + if (mode === 'antora') { + // Legacy path: run Antora lunr extension once + const indexOut = path.join(docsSiteDir, 'build', '.fanout', 'index') + let indexPb = JSON.parse(JSON.stringify(playbook)) + if (indexPb.antora && Array.isArray(indexPb.antora.extensions)) { + const hasLunr = indexPb.antora.extensions.some((e) => e && e.require === '@antora/lunr-extension') + if (!hasLunr) indexPb.antora.extensions.unshift({ require: '@antora/lunr-extension', index_latest_only: true }) + } + indexPb = rebasePlaybookPaths(indexPb, docsSiteDir) + indexPb.output = indexPb.output || {} + indexPb.output.dir = path.relative(docsSiteDir, indexOut) + const indexPlaybookFile = path.join(fanoutRoot, 'playbook-index.yml') + await writeYaml(indexPlaybookFile, indexPb) + console.log('[fanout] Building global Lunr index with Antora (legacy)...') + try { + await runAntora(docsSiteDir, indexPlaybookFile) + // Copy search-index artifacts from indexOut into finalOut + const entries = await fsp.readdir(indexOut) + const indexFiles = entries.filter((name) => /^search-index\..*/.test(name)) + for (const name of indexFiles) { + const src = path.join(indexOut, name) + const dst = path.join(finalOut, name) + await mkdirp(path.dirname(dst)) + await fsp.copyFile(src, dst) + } + if (indexFiles.length) { + console.log(`[fanout] Copied global Lunr artifacts to final site: ${indexFiles.join(', ')}`) + } else { + console.warn('[fanout] No search-index.* artifacts found to copy. Verify lunr extension configuration.') + } + } catch (e) { + console.error('[fanout] Global Lunr index build failed (Antora):', e && e.message ? e.message : e) + process.exitCode = 1 + return + } + } else { + // Fast path: build directly from merged site + const indexer = path.join(__dirname, 'build-lunr-from-site.js') + console.log('[fanout] Building global Lunr index from merged site (fast path)...') + try { + // invoke Node child process to avoid leaking state + await new Promise((resolve, reject) => { + const env = { ...process.env } + // Use ultra-conservative defaults to reduce memory pressure; can be overridden via env + if (!env.ANTORA_LUNR_IO_WORKERS) env.ANTORA_LUNR_IO_WORKERS = '1' + if (!env.ANTORA_LUNR_SIMPLE_PIPELINE) env.ANTORA_LUNR_SIMPLE_PIPELINE = '1' + // Allow increasing V8 heap for large index serialization + const maxOldSpace = env.ANTORA_LUNR_NODE_MAX_OLD_SPACE || '6144' + const existingNodeOptions = env.NODE_OPTIONS ? `${env.NODE_OPTIONS} ` : '' + env.NODE_OPTIONS = `${existingNodeOptions}--max-old-space-size=${maxOldSpace}`.trim() + const child = require('child_process').spawn(process.execPath, [indexer, path.join(docsSiteDir, 'build', 'site')], { + cwd: docsSiteDir, + stdio: 'inherit', + env, + }) + child.on('exit', (code) => (code === 0 ? resolve() : reject(new Error(`indexer exited with code ${code}`)))) + child.on('error', reject) + }) + } catch (e) { + console.warn('[fanout] Fast Lunr indexer failed; falling back to Antora:', e && e.message ? e.message : e) + process.env.ANTORA_LUNR_MODE = 'antora' + return await (async () => { // recurse into legacy path + const indexOut = path.join(docsSiteDir, 'build', '.fanout', 'index') + let indexPb = JSON.parse(JSON.stringify(playbook)) + if (indexPb.antora && Array.isArray(indexPb.antora.extensions)) { + const hasLunr = indexPb.antora.extensions.some((e) => e && e.require === '@antora/lunr-extension') + if (!hasLunr) indexPb.antora.extensions.unshift({ require: '@antora/lunr-extension', index_latest_only: true }) + } + indexPb = rebasePlaybookPaths(indexPb, docsSiteDir) + indexPb.output = indexPb.output || {} + indexPb.output.dir = path.relative(docsSiteDir, indexOut) + const indexPlaybookFile = path.join(fanoutRoot, 'playbook-index.yml') + await writeYaml(indexPlaybookFile, indexPb) + await runAntora(docsSiteDir, indexPlaybookFile) + const entries = await fsp.readdir(indexOut) + const indexFiles = entries.filter((name) => /^search-index\..*/.test(name)) + for (const name of indexFiles) { + const src = path.join(indexOut, name) + const dst = path.join(finalOut, name) + await mkdirp(path.dirname(dst)) + await fsp.copyFile(src, dst) + } + })() + } + } + } else { + console.log('[fanout] Lunr disabled (--no-lunr). Skipping global index pass.') + } +} + +main().catch((e) => { + console.error('[fanout] fatal:', e && e.message ? e.message : e) + process.exit(1) +}) diff --git a/docs-site/extensions/antora-sources-parallel/index.js b/docs-site/extensions/antora-sources-parallel/index.js new file mode 100644 index 0000000000..b0b9c72467 --- /dev/null +++ b/docs-site/extensions/antora-sources-parallel/index.js @@ -0,0 +1,66 @@ +/* + * Antora extension: antora-sources-parallel + * Goal: Provide an extension hook and configuration to enable per-source/module parallel execution. + * Phase 1: Safe concurrency activation via environment hints (fetch/build), with worker count policy. + * Phase 2 (future): Optional fan-out per-source builds and merge (experimental, off by default). + */ + +const os = require('os') + +function computeWorkers (min = 3) { + const cores = Array.isArray(os.cpus()) ? os.cpus().length : 1 + return Math.max(min, cores || 1) +} + +module.exports = function (registry) { + const logger = console + + registry.on('playbookBuilt', ({ playbook }) => { + try { + const raw = playbook && (playbook.asMutable?.() || playbook) + const extensions = raw?.antora?.extensions || [] + const selfEntry = Array.isArray(extensions) + ? extensions.find((e) => (e && (e.require === 'antora-sources-parallel' || e.require === './extensions/antora-sources-parallel' || e.require === './extensions/antora-sources-parallel/index.js' || e.require === './js/sources-parallel.js'))) + : undefined + + const enabled = !!(selfEntry && (selfEntry.sources_parallel === true || selfEntry.enabled === true)) + const minWorkers = typeof selfEntry?.min_workers === 'number' ? selfEntry.min_workers : 3 + const configuredMax = typeof selfEntry?.max_workers === 'number' ? selfEntry.max_workers : undefined + + const autoWorkers = computeWorkers(minWorkers) + const workers = configuredMax && configuredMax > 0 ? configuredMax : autoWorkers + + if (!enabled) { + logger.log(`[antora-sources-parallel] Loaded (disabled). Computed workers=${workers}. No changes applied.`) + return + } + + // Expose workers via env to encourage Antora and other extensions to leverage concurrency + // These envs are de-facto hints; if Antora honors them, great; otherwise harmless. + const envHints = [ + 'ANTORA_FETCH_CONCURRENCY', // potential fetch concurrency + 'ANTORA_CONCURRENCY', // generic task concurrency + 'ANTORA_SOURCES_PARALLEL_WORKERS', // our own explicit value for cooperating tools + ] + for (const key of envHints) { + if (!process.env[key]) process.env[key] = String(workers) + } + + // Encourage Antora cache reuse in CI/local unless already specified + if (!process.env.ANTORA_CACHE_DIR) { + // Use a project-local cache directory so CI can persist it as an artifact between runs + process.env.ANTORA_CACHE_DIR = require('path').join(process.cwd(), 'build', '.cache') + } + + logger.log(`[antora-sources-parallel] Enabled with workers=${workers}. Concurrency env hints set: ${envHints.join(', ')}. Cache dir: ${process.env.ANTORA_CACHE_DIR}`) + + // Experimental fan-out guidance + const experimental = selfEntry?.experimental_fanout === true + if (experimental) { + logger.warn('[antora-sources-parallel] Experimental fan-out enabled in playbook. To run per-source fan-out, execute: npm run antora-fanout (from docs-site). Default antora run remains single-process (env-hint mode only).') + } + } catch (e) { + logger.warn(`[antora-sources-parallel] Failed to initialize: ${e && e.message ? e.message : e}`) + } + }) +} diff --git a/docs-site/extensions/antora-sources-parallel/package.json b/docs-site/extensions/antora-sources-parallel/package.json new file mode 100644 index 0000000000..3c094be4d2 --- /dev/null +++ b/docs-site/extensions/antora-sources-parallel/package.json @@ -0,0 +1,24 @@ +{ + "name": "antora-sources-parallel", + "version": "0.1.0", + "description": "Antora extension to enable parallel-friendly builds via concurrency hints and future per-source fan-out.", + "main": "index.js", + "license": "Apache-2.0", + "keywords": [ + "antora", + "extension", + "parallel", + "concurrency", + "docs" + ], + "bin": { + "antora-fanout": "bin/fanout.js" + }, + "dependencies": { + "js-yaml": "^4.1.0", + "lunr": "^2.3.9" + }, + "engines": { + "node": ">=14" + } +} diff --git a/docs-site/js/sources-parallel.js b/docs-site/js/sources-parallel.js new file mode 100644 index 0000000000..989140c9cc --- /dev/null +++ b/docs-site/js/sources-parallel.js @@ -0,0 +1,54 @@ +/* + * Antora extension: sources-parallel + * Goal: Provide an extension hook and configuration to enable per-source/module parallel execution. + * Note: This initial scaffold computes worker count and logs activation; it does not yet override the + * generator pipeline. It is designed to remain safe/inert while we iterate on true parallelism. + */ + +const os = require('os') + +function computeWorkers (min = 3) { + const cores = Array.isArray(os.cpus()) ? os.cpus().length : 1 + return Math.max(min, cores || 1) +} + +module.exports = function (registry) { + // Antora passes the playbook to hooks; we use playbookBuilt to read extension options + const logger = console + + registry.on('playbookBuilt', ({ playbook }) => { + try { + // Locate our extension options if provided via the playbook extensions entry + // When listed as: { require: './js/sources-parallel.js', sources_parallel: true, max_workers: N } + // Antora makes extension options available via playbook.get('antora.extensions') for some APIs; + // since this is not a public API surface, we defensively parse from the raw object when possible. + const raw = playbook && (playbook.asMutable?.() || playbook) + const extensions = raw?.antora?.extensions || [] + const selfEntry = Array.isArray(extensions) + ? extensions.find((e) => (e && (e.require === './js/sources-parallel.js' || e.require === 'js/sources-parallel.js'))) + : undefined + + const enabled = !!(selfEntry && (selfEntry.sources_parallel === true || selfEntry.enabled === true)) + const minWorkers = typeof selfEntry?.min_workers === 'number' ? selfEntry.min_workers : 3 + const configuredMax = typeof selfEntry?.max_workers === 'number' ? selfEntry.max_workers : undefined + + const autoWorkers = computeWorkers(minWorkers) + const workers = configuredMax && configuredMax > 0 ? configuredMax : autoWorkers + + if (!enabled) { + logger.log(`[sources-parallel] Loaded (disabled). Computed workers=${workers}. No changes applied.`) + return + } + + // Expose workers via env to allow future cooperation with other extensions/tools + if (!process.env.ANTORA_SOURCES_PARALLEL_WORKERS) { + process.env.ANTORA_SOURCES_PARALLEL_WORKERS = String(workers) + } + + // Placeholder: future iterations will patch the generator to fan-out per-source work. + logger.log(`[sources-parallel] Enabled with workers=${workers}. Parallel generation hooks not yet active (scaffold).`) + } catch (e) { + logger.warn(`[sources-parallel] Failed to initialize: ${e && e.message ? e.message : e}`) + } + }) +} diff --git a/docs-site/package.json b/docs-site/package.json index 0be81fee14..46f2d774fe 100644 --- a/docs-site/package.json +++ b/docs-site/package.json @@ -7,7 +7,12 @@ "escape-string-regexp": "^5.0.0", "pako": "^2.1.0" }, + "dependencies": { + "antora-sources-parallel": "file:./extensions/antora-sources-parallel" + }, "scripts": { - "npx": "npx" + "npx": "npx", + "antora-fanout": "node ./extensions/antora-sources-parallel/bin/fanout.js antora-playbook.yml", + "antora:ci": "node ./extensions/antora-sources-parallel/bin/fanout.js antora-playbook.yml --no-lunr" } }