diff --git a/.github/workflows/publish_readme-docs.yml b/.github/workflows/publish_readme-docs.yml new file mode 100644 index 000000000000..d1434b1e587a --- /dev/null +++ b/.github/workflows/publish_readme-docs.yml @@ -0,0 +1,276 @@ +# Publish README Docs +# +# Copies markdown files from this repo (mostly SDK and example READMEs) into their +# documentation pages (DotcmsDocumentation contentlets) on the dotCMS authoring server, so +# dev.dotcms.com serves and indexes the same text instead of mirroring it at request time. +# The file is the single source of truth: each run overwrites the page body. dotCMS keeps +# the version history. +# +# Which pages: every DotcmsDocumentation page whose `readmeSource` field holds a URL of the +# form https://github.com/dotCMS/core/blob/main/.md. Pages are added or removed by +# editing that field in dotCMS; no change to this workflow is needed. URLs pointing anywhere +# else are left alone. +# +# Each changed page goes through the authoring server's Push Publish workflow action (save, +# publish, unlock, push to the `prod` environment), and the run waits for that push to +# succeed. The token belongs to the CI Autodoc Publish User, which needs the Dev Publish role +# on the site, on the Documentation content type, on the Push Publish action and on the +# `prod` push environment. +# +# Runs on every merge that touches a markdown file, and manually (dry run by default). +# Every run checks all mapped pages and writes only those whose body differs. + +name: Publish README Docs + +on: + push: + branches: [main] + paths: + - '**/*.md' + workflow_dispatch: + inputs: + dry_run: + description: 'Compare only and show the diffs in the run summary; write nothing to dotCMS' + type: boolean + default: true + pages: + description: 'Space-separated page urlTitles to sync (empty = all mapped pages)' + type: string + default: '' + +concurrency: + group: publish-readme-docs + cancel-in-progress: false + +permissions: + contents: read + +env: + # GitHub users DMed by dotBot when a sync fails (looked up in .github/data/slack-mappings.json). + NOTIFY_GITHUB_USERS: 'jdcmsd' + +jobs: + sync: + name: Sync READMEs to dotCMS + runs-on: ubuntu-${{ vars.UBUNTU_RUNNER_VERSION || '24.04' }} + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + + - name: Sync pages + env: + DOTCMS_URL: ${{ vars.AUTODOC_DOTCMS_BASE_URL }} + DOTCMS_TOKEN: ${{ secrets.AUTODOC_DOTCMS_API_TOKEN_PUBLISH }} + # Empty on push, so merges always write. + DRY_RUN: ${{ inputs.dry_run }} + PAGES_FILTER: ${{ inputs.pages }} + run: | + python3 - <<'PY' + import difflib, json, os, posixpath, re, sys, time, urllib.error, urllib.request + + REPO = "dotCMS/core" + # Authoring server's "Push Publish" action: Save, Publish, Unlock, Push/Remove Now (to `prod`). + PUSH_PUBLISH_ACTION = "0c5239e8-4aef-4dcd-b31f-ab5ca0394ea2" + PUSH_TIMEOUT = 180 # seconds to wait for a page's push bundle to succeed + SOURCE_PREFIX = f"https://github.com/{REPO}/blob/main/" + LINK_BASE = f"https://github.com/{REPO}/blob/main" + ASSET_BASE = f"https://raw.githubusercontent.com/{REPO}/main" + # Working versions of mapped pages. size is well above the number of pages. + PAGE_QUERY = ("+contenttype:dotcmsdocumentation +working:true +languageid:1 +deleted:false " + "+dotcmsdocumentation.readmesource:*") + + base = os.environ["DOTCMS_URL"].rstrip("/") + token = os.environ["DOTCMS_TOKEN"] + dry_run = os.environ.get("DRY_RUN") == "true" + selected = set(os.environ.get("PAGES_FILTER", "").split()) + summary = open(os.environ["GITHUB_STEP_SUMMARY"], "a") + + # --- README clean-up, matching the devsite's former live mirror --- + NOT_ABSOLUTE = r"(?!https?://)(?!/)(?!#)(?!data:)(?!mailto:)(?!tel:)(?:\./)?" + + def outside_fences(lines): + """Indexes of lines not inside ``` / ~~~ code blocks.""" + fence = None + for i, line in enumerate(lines): + m = re.match(r"\s*(```|~~~)", line) + if m: + fence = None if fence == m[1] else (fence or m[1]) + elif fence is None: + yield i + + def clean(readme, path): + lines = readme.replace("\r\n", "\n").split("\n") + # Drop a "Table of Contents" section: its heading up to the next heading. + heads = [i for i in outside_fences(lines) if re.match(r"#+\s", lines[i])] + toc = next((i for i in heads if re.search(r"table\s+of\s+contents", lines[i], re.I)), None) + if toc is not None: + end = next((i for i in heads if i > toc), len(lines)) + del lines[toc:end] + # Drop the first H1; the page renders its own title. + h1 = next((i for i in outside_fences(lines) if re.match(r"#\s", lines[i])), None) + if h1 is not None: + del lines[h1] + text = "\n".join(lines) + + # Relative links and images would resolve against dev.dotcms.com; point them at GitHub. + directory = posixpath.dirname(path) + def absolute(ref): + target, sep, anchor = ref.partition("#") + resolved = posixpath.normpath(posixpath.join(directory, target)) if target else directory + return resolved + sep + anchor + text = re.sub(r"!\[([^\]]*)\]\(" + NOT_ABSOLUTE + r"([^)]+)\)", + lambda m: f"![{m[1]}]({ASSET_BASE}/{absolute(m[2])})", text) + text = re.sub(r"(?]*)\s+src=[\"']" + NOT_ABSOLUTE + r"([^\"']+)[\"']", + lambda m: f'= 500: # dotCMS intermittently fails a save; a retry has always gone through + status, data = call("PUT", url, payload) + if status != 200 or data.get("errors"): + # A failed save can leave the page locked to this account; release it. + call("PUT", f"/api/v1/content/_unlock/{ident}?language=1") + raise RuntimeError(f"save/publish failed: HTTP {status} {data.get('errors') or data}") + return wait_for_push(ident, before) + + # A bad token still reads the public pages as anonymous, so check it before anything else. + status, data = call("GET", "/api/v1/users/current/") + if status != 200: + sys.exit(f"dotCMS token rejected: HTTP {status} {data}") + + status, data = call("POST", "/api/es/search", {"query": {"query_string": {"query": PAGE_QUERY}}, "size": 1000}) + pages = data.get("contentlets") if status == 200 else None + if not pages: + # Never a valid state; most likely a token, permission or search index problem. + sys.exit(f"no mapped pages found (HTTP {status}): {data if status != 200 else 'empty result'}") + unknown = selected - {p.get("urlTitle") for p in pages} + if unknown: + sys.exit(f"not a mapped page urlTitle: {' '.join(sorted(unknown))}") + + rows, diffs, failed = [], [], 0 + for page in sorted(pages, key=lambda p: p.get("urlTitle") or ""): + slug, ident, url = page.get("urlTitle"), page["identifier"], page.get("readmeSource", "").strip() + if selected and slug not in selected: + continue + path = None + try: + path = source_path(url) + if path is None: + result = "other source, skipped" + else: + body = clean(open(path, encoding="utf-8").read(), path) + # Search results can lag behind recent saves; compare with the page itself. + status, data = call("GET", f"/api/v1/content/{ident}") + if status != 200: + raise RuntimeError(f"read failed: HTTP {status} {data}") + current = data["entity"].get("documentation") or "" + if normalize(current) == normalize(body): + result = "unchanged" + elif dry_run: + result = "would update" + diffs.append((slug, "".join(difflib.unified_diff( + current.splitlines(True), body.splitlines(True), "dotCMS", path)))) + else: + result = f"published and pushed (bundle {push_publish(ident, body)})" + except Exception as exc: + failed += 1 + result = f"FAILED: {exc}" + print(f"{slug:24} {result}") + rows.append(f"| `{slug}` | {f'`{path}`' if path else url} | {result} |") + + summary.write(f"## README docs sync{' (dry run)' if dry_run else ''}\n\n| Page | Source | Result |\n|---|---|---|\n") + summary.write("\n".join(rows) + "\n") + for slug, diff in diffs: + summary.write(f"\n
{slug}\n\n```diff\n{diff}\n```\n
\n") + sys.exit(1 if failed else 0) + PY + + - name: Notify on failure + if: failure() + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + text=":x: README docs sync failed. ${RUN_URL}" + for user in $NOTIFY_GITHUB_USERS; do + slack_id=$(jq -r --arg u "$user" '.[] | select(.github_user == $u) | .slack_id' .github/data/slack-mappings.json) + if [ -z "$slack_id" ]; then + echo "::warning::No Slack mapping for GitHub user ${user}" + continue + fi + jq -n --arg c "$slack_id" --arg t "$text" '{channel: $c, text: $t}' \ + | curl -s -X POST https://slack.com/api/chat.postMessage \ + -H "Content-type: application/json; charset=utf-8" \ + -H "Authorization: Bearer ${SLACK_BOT_TOKEN}" \ + --data @- \ + | jq -e '.ok' > /dev/null || echo "::warning::Slack DM to ${user} failed" + done