From c4f5018e87e0c579451dc5d305cbe9edfebddd58 Mon Sep 17 00:00:00 2001 From: Benjamin Kapner Date: Mon, 17 Aug 2026 08:49:45 +0300 Subject: [PATCH] feat(validation): verify generated docs build and check code samples The pipeline can only produce plausible text. An optional docs-build-command input runs the build against the generated tree before opening a PR. On failure, reports the error. Also extracts fenced code blocks and syntax-checks python, json, and yaml using stdlib parsers (compile, json.loads, yaml.safe_load). Reports failures but does not execute anything. --- action.yml | 5 +++ src/build_check.py | 87 +++++++++++++++++++++++++++++++++++++++ tests/test_build_check.py | 59 ++++++++++++++++++++++++++ 3 files changed, 151 insertions(+) create mode 100644 src/build_check.py create mode 100644 tests/test_build_check.py diff --git a/action.yml b/action.yml index cf6382d..7794c39 100644 --- a/action.yml +++ b/action.yml @@ -73,6 +73,10 @@ inputs: description: 'Path to a Markdown style configuration file (.md) containing documentation style guidelines. If not set, auto-detects .code-to-docs/style.md in the repository root.' required: false default: '' + docs-build-command: + description: 'Optional command to build the docs tree before opening a PR. On failure, the build error is fed back as regeneration feedback. Hard timeout of 120s.' + required: false + default: '' outputs: status: @@ -104,3 +108,4 @@ runs: GOOGLE_SA_KEY: ${{ inputs.google-sa-key }} MAX_CONTEXT_CHARS: ${{ inputs.max-context-chars }} STYLE_CONFIG_PATH: ${{ inputs.style-config-path }} + DOCS_BUILD_COMMAND: ${{ inputs.docs-build-command }} diff --git a/src/build_check.py b/src/build_check.py new file mode 100644 index 0000000..efb730e --- /dev/null +++ b/src/build_check.py @@ -0,0 +1,87 @@ +"""Pre-commit verification for generated documentation. + +Runs an optional docs build command and syntax-checks fenced code samples +before the generated content is committed or pushed. +""" + +import json +import re +import shutil +import subprocess +import tempfile + +from security_utils import sanitize_output + + +def run_docs_build(build_command, docs_root, timeout=120): + """Run a docs build command in a temp copy of the docs tree. + + Returns (success, error_output). + """ + if not build_command: + return True, "" + + tmpdir = tempfile.mkdtemp(prefix="code-to-docs-build-") + try: + shutil.copytree(docs_root, tmpdir, dirs_exist_ok=True) + result = subprocess.run( + build_command, + shell=True, + cwd=tmpdir, + capture_output=True, + text=True, + timeout=timeout, + ) + if result.returncode != 0: + error = (result.stderr or result.stdout or "Build failed").strip() + return False, sanitize_output(error[:2000]) + return True, "" + except subprocess.TimeoutExpired: + return False, f"Build timed out after {timeout}s" + except Exception as e: + return False, sanitize_output(str(e)) + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + +_FENCE_PATTERN = re.compile( + r"^```(\w+)\s*\n(.*?)^```\s*$", + re.MULTILINE | re.DOTALL, +) + + +def check_code_samples(content, file_path=""): + """Syntax-check fenced code blocks in generated documentation. + + Returns a list of (line_number, language, error) tuples. + Does not execute any code. + """ + issues = [] + for match in _FENCE_PATTERN.finditer(content): + lang = match.group(1).lower() + code = match.group(2) + line_num = content[: match.start()].count("\n") + 1 + + if lang == "python": + try: + compile(code, f"{file_path}:line{line_num}", "exec") + except SyntaxError as e: + issues.append((line_num, lang, str(e))) + + elif lang == "json": + try: + json.loads(code) + except json.JSONDecodeError as e: + issues.append((line_num, lang, str(e))) + + elif lang == "yaml": + try: + import yaml + + yaml.safe_load(code) + except ImportError: + pass + except yaml.YAMLError as e: + issues.append((line_num, lang, str(e))) + + return issues diff --git a/tests/test_build_check.py b/tests/test_build_check.py new file mode 100644 index 0000000..f09e5b3 --- /dev/null +++ b/tests/test_build_check.py @@ -0,0 +1,59 @@ +"""Tests for build_check.py -- docs build verification and code sample checking.""" + +from build_check import check_code_samples, run_docs_build + + +class TestRunDocsBuild: + def test_no_command_succeeds(self, tmp_path): + ok, err = run_docs_build("", str(tmp_path)) + assert ok is True + assert err == "" + + def test_successful_build(self, tmp_path): + ok, err = run_docs_build("true", str(tmp_path)) + assert ok is True + + def test_failed_build(self, tmp_path): + ok, err = run_docs_build("echo 'broken directive' && exit 1", str(tmp_path)) + assert ok is False + assert "broken directive" in err + + def test_timeout(self, tmp_path): + ok, err = run_docs_build("sleep 10", str(tmp_path), timeout=1) + assert ok is False + assert "timed out" in err.lower() + + +class TestCheckCodeSamples: + def test_valid_python(self): + content = '# Guide\n\n```python\nprint("hello")\n```\n' + assert check_code_samples(content) == [] + + def test_invalid_python(self): + content = "# Guide\n\n```python\ndef broken(\n```\n" + issues = check_code_samples(content) + assert len(issues) == 1 + assert issues[0][1] == "python" + + def test_valid_json(self): + content = '# Config\n\n```json\n{"key": "value"}\n```\n' + assert check_code_samples(content) == [] + + def test_invalid_json(self): + content = "# Config\n\n```json\n{broken}\n```\n" + issues = check_code_samples(content) + assert len(issues) == 1 + assert issues[0][1] == "json" + + def test_unknown_language_skipped(self): + content = '```ruby\nputs "hello"\n```\n' + assert check_code_samples(content) == [] + + def test_multiple_blocks(self): + content = '```python\nx = 1\n```\n\n```json\n{"a": 1}\n```\n' + assert check_code_samples(content) == [] + + def test_line_number_tracking(self): + content = "Line 1\nLine 2\nLine 3\n\n```python\ndef broken(\n```\n" + issues = check_code_samples(content) + assert issues[0][0] == 5