From 93e9a6176a96afc5c0a62f9ea662dd323062d750 Mon Sep 17 00:00:00 2001 From: rangamani54 Date: Sat, 15 Aug 2026 13:41:03 +0530 Subject: [PATCH] feat: add Jenkinsfile pipeline extractor --- README.md | 1 + graphify/detect.py | 4 + graphify/extract.py | 9 +- graphify/extractors/__init__.py | 2 + graphify/extractors/jenkins.py | 416 ++++++++++++++++++++++++++++++++ tests/test_jenkins.py | 122 ++++++++++ 6 files changed, 552 insertions(+), 2 deletions(-) create mode 100644 graphify/extractors/jenkins.py create mode 100644 tests/test_jenkins.py diff --git a/README.md b/README.md index bf09a07f8..33a3e69cd 100644 --- a/README.md +++ b/README.md @@ -339,6 +339,7 @@ To remove graphify from all platforms at once: `graphify uninstall` (add `--purg | Code (37 tree-sitter grammars) | `.py .ts .mts .cts .js .jsx .tsx .mjs .go .rs .java .c .cpp .cc .cxx .h .hpp .cu .cuh .metal .rb .cs .kt .kts .scala .php .swift .lua .luau .toc .zig .ps1 .psm1 .psd1 .ex .exs .m .mm .ml .mli .jl .vue .svelte .astro .groovy .gradle .dart .v .sv .svh .sql .f .f90 .f95 .f03 .f08 .pas .pp .dpr .dpk .lpr .inc .dfm .lfm .lpk .sh .bash .json .dm .dme .dmi .dmm .dmf .sln .slnx .csproj .fsproj .vbproj .xaml .razor .cshtml` (`.dm`/`.dme` requires `uv tool install graphifyy[dm]`, `.ml`/`.mli` requires `uv tool install graphifyy[ocaml]`; `.mts`/`.cts` reuse the TypeScript grammar, `.cc`/`.cxx` and CUDA `.cu`/`.cuh` and Metal `.metal` reuse the C++ grammar) | | Salesforce Apex | `.cls .trigger` (regex-based; classes, interfaces, enums, methods, triggers, SOQL/DML edges) | | Terraform / HCL | `.tf .tfvars .hcl` (requires `uv tool install graphifyy[terraform]`) | +| Jenkins Pipeline | `Jenkinsfile` (Groovy DSL; extracts pipelines, stages, parallel branches, steps, functions, and Docker image facts) | | OCaml | `.ml .mli` (requires `uv tool install graphifyy[ocaml]`) | | MCP configs | `.mcp.json` `mcp.json` `mcp_servers.json` `claude_desktop_config.json` — extracts server nodes, package refs, env var requirements | | Package manifests | `apm.yml` `pyproject.toml` `go.mod` `pom.xml` — one canonical package node per package (by name) plus `depends_on` edges, so a package referenced from many manifests is a single hub | diff --git a/graphify/detect.py b/graphify/detect.py index c51ea916e..e5a28d9b7 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -508,6 +508,10 @@ def classify_file(path: Path) -> FileType | None: from graphify.manifest_ingest import is_package_manifest_path if is_package_manifest_path(path): return FileType.CODE + # Jenkins Pipeline files are conventionally extensionless, so they need a + # filename-based route before the generic shebang/suffix checks. + if path.name.lower() == "jenkinsfile": + return FileType.CODE # Compound extensions must be checked before simple suffix lookup if path.name.lower().endswith(".blade.php"): return FileType.CODE diff --git a/graphify/extract.py b/graphify/extract.py index 1822c556f..a02c683b5 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -45,6 +45,7 @@ from graphify.extractors.fortran import _cpp_preprocess, extract_fortran # noqa: F401 from graphify.extractors.go import _GO_PREDECLARED_FUNCS, extract_go # noqa: F401 from graphify.extractors.json_config import extract_json # noqa: F401 +from graphify.extractors.jenkins import extract_jenkinsfile # noqa: F401 from graphify.extractors.markdown import extract_markdown # noqa: F401 from graphify.extractors.ocaml import extract_ocaml # noqa: F401 from graphify.extractors.pascal_forms import extract_delphi_form, extract_lazarus_form # noqa: F401 @@ -4989,6 +4990,8 @@ def _is_cpp_header(path: Path) -> bool: def _get_extractor(path: Path) -> Any | None: """Return the correct extractor function for a file, or None if unsupported.""" + if path.name.lower() == "jenkinsfile": + return extract_jenkinsfile if path.name.lower().endswith(".blade.php"): return extract_blade # MCP config files (.mcp.json, claude_desktop_config.json, ...) are routed @@ -6685,7 +6688,8 @@ def _ignored(p: Path) -> bool: for fname in filenames: p = dp / fname suffix = p.suffix - if (suffix in _EXTENSIONS or suffix.lower() in _EXTENSIONS) and not _ignored(p) and _resolves_under_root(p, containment_root): + is_jenkinsfile = p.name.lower() == "jenkinsfile" + if (is_jenkinsfile or suffix in _EXTENSIONS or suffix.lower() in _EXTENSIONS) and not _ignored(p) and _resolves_under_root(p, containment_root): results.append(p) return sorted(results) # Walk with symlink following + cycle detection @@ -6706,7 +6710,8 @@ def _ignored(p: Path) -> bool: for fname in filenames: p = dp / fname suffix = p.suffix - if (suffix in _EXTENSIONS or suffix.lower() in _EXTENSIONS) and not _ignored(p) and _resolves_under_root(p, containment_root): + is_jenkinsfile = p.name.lower() == "jenkinsfile" + if (is_jenkinsfile or suffix in _EXTENSIONS or suffix.lower() in _EXTENSIONS) and not _ignored(p) and _resolves_under_root(p, containment_root): results.append(p) return sorted(results) diff --git a/graphify/extractors/__init__.py b/graphify/extractors/__init__.py index ada517094..60ed20c4e 100644 --- a/graphify/extractors/__init__.py +++ b/graphify/extractors/__init__.py @@ -19,6 +19,7 @@ from graphify.extractors.fortran import extract_fortran from graphify.extractors.go import extract_go from graphify.extractors.json_config import extract_json +from graphify.extractors.jenkins import extract_jenkinsfile from graphify.extractors.julia import extract_julia from graphify.extractors.markdown import extract_markdown from graphify.extractors.objc import extract_objc @@ -47,6 +48,7 @@ "fortran": extract_fortran, "go": extract_go, "json": extract_json, + "jenkinsfile": extract_jenkinsfile, "julia": extract_julia, "lazarus_form": extract_lazarus_form, "markdown": extract_markdown, diff --git a/graphify/extractors/jenkins.py b/graphify/extractors/jenkins.py new file mode 100644 index 000000000..af4ec868b --- /dev/null +++ b/graphify/extractors/jenkins.py @@ -0,0 +1,416 @@ +"""Jenkins Pipeline extractor for extensionless ``Jenkinsfile`` sources.""" +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from graphify.extractors.base import _make_id + + +_STRUCTURAL_BLOCKS = frozenset({ + "agent", + "environment", + "matrix", + "options", + "parameters", + "parallel", + "post", + "stages", + "tools", + "triggers", + "when", +}) +_STAGE_CONTAINER_NAMES = frozenset({"stage", "stages"}) +_IMAGE_CALLS = frozenset({"image", "docker.image", "docker.build"}) +_SHARED_LIBRARY_ANNOTATION = "Library" +_PARALLEL_BLOCKS = frozenset({"parallel", "matrix"}) + + +def _line(node: Any) -> str: + return f"L{node.start_point[0] + 1}" + + +def extract_jenkinsfile(path: Path) -> dict: + """Extract Jenkins Pipeline, stage, step, and Docker image facts. + + The Jenkinsfile is parsed as Groovy, but its useful graph structure comes + from the Jenkins Pipeline DSL rather than Groovy classes and methods. We + therefore keep the regular Groovy extractor unchanged and walk Pipeline + method invocations separately. Any call inside a ``steps`` block is + treated as a step, which covers both Jenkins built-ins and shared-library + steps without maintaining a brittle allow-list. + """ + try: + import tree_sitter_groovy as tsgroovy + from tree_sitter import Language, Parser + + parser = Parser(Language(tsgroovy.language())) + source = path.read_bytes() + tree = parser.parse(source) + except ImportError: + return { + "nodes": [], + "edges": [], + "error": "tree_sitter_groovy not installed. Run: pip install tree-sitter-groovy", + } + except Exception as exc: + return {"nodes": [], "edges": [], "error": str(exc)} + + root = tree.root_node + str_path = str(path) + file_nid = _make_id(str_path) + stem = path.with_suffix("").as_posix() if path.name else str(path) + nodes: list[dict] = [{ + "id": file_nid, + "label": path.name, + "file_type": "code", + "type": "jenkinsfile", + "source_file": str_path, + "source_location": None, + }] + edges: list[dict] = [] + seen_nodes: set[str] = {file_nid} + seen_edges: set[tuple[str, str, str]] = set() + stage_number = 0 + step_number = 0 + parallel_number = 0 + image_ids: dict[str, str] = {} + library_ids: dict[str, str] = {} + function_ids: dict[str, str] = {} + pipeline_nid: str | None = None + + def read(node: Any) -> str: + return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace") + + def add_edge(src: str, target: str, relation: str, node: Any) -> None: + key = (src, target, relation) + if key in seen_edges: + return + seen_edges.add(key) + edges.append({ + "source": src, + "target": target, + "relation": relation, + "confidence": "EXTRACTED", + "source_file": str_path, + "source_location": _line(node), + "weight": 1.0, + }) + + def add_node(nid: str, label: str, node: Any, node_type: str) -> None: + if nid in seen_nodes: + return + seen_nodes.add(nid) + nodes.append({ + "id": nid, + "label": label, + "file_type": "code", + "type": node_type, + "source_file": str_path, + "source_location": _line(node), + }) + + def call_name(node: Any) -> str | None: + # Groovy's grammar can parse the common Jenkins DSL form + # ``checkout scm`` as a local-variable declaration when it appears on + # a newline without parentheses. In a ``steps`` block the declared + # type is the Pipeline step name, so retain it as a step fact. + if node.type == "local_variable_declaration": + type_node = next( + (child for child in node.named_children if child.type == "type_identifier"), + None, + ) + return read(type_node).strip() if type_node is not None else None + if node.type not in {"method_invocation", "juxt_function_call"}: + return None + name_node = node.child_by_field_name("name") + if name_node is None: + return None + name = read(name_node).strip() + object_node = node.child_by_field_name("object") + if object_node is not None: + name = f"{read(object_node).strip()}.{name}" + return name or None + + def literal_value(node: Any) -> str | None: + if node.type not in { + "character_literal", + "string_literal", + "interpolated_string", + "string_content", + }: + return None + value = read(node).strip() + if len(value) >= 2 and value[0] in "'\"" and value[-1] == value[0]: + return value[1:-1] + return value or None + + def first_literal(node: Any) -> str | None: + value = literal_value(node) + if value is not None: + return value + for child in node.named_children: + value = first_literal(child) + if value is not None: + return value + return None + + def call_argument(node: Any) -> str | None: + arguments = ( + node.child_by_field_name("arguments") + or node.child_by_field_name("args") + ) + return first_literal(arguments) if arguments is not None else None + + def body(node: Any) -> Any | None: + return node.child_by_field_name("body") + + def add_image(image_ref: str, owner: str, relation: str, node: Any) -> None: + image_ref = image_ref.strip() + if not image_ref: + return + image_nid = image_ids.get(image_ref) + if image_nid is None: + image_nid = _make_id(stem, "docker_image", image_ref) + image_ids[image_ref] = image_nid + add_node(image_nid, image_ref, node, "docker_image") + add_edge(file_nid, image_nid, "contains", node) + add_edge(owner, image_nid, relation, node) + + def add_library(library_ref: str, node: Any) -> None: + library_ref = library_ref.strip() + if not library_ref: + return + library_nid = library_ids.get(library_ref) + if library_nid is None: + library_nid = _make_id(stem, "shared_library", library_ref) + library_ids[library_ref] = library_nid + add_node(library_nid, library_ref, node, "jenkins_shared_library") + add_edge(file_nid, library_nid, "contains", node) + if pipeline_nid is not None: + add_edge(pipeline_nid, library_nid, "uses_library", node) + + def add_pipeline(node: Any) -> str: + nonlocal pipeline_nid + if pipeline_nid is None: + pipeline_nid = _make_id(stem, "pipeline") + add_node(pipeline_nid, "JenkinsPipeline", node, "jenkins_pipeline") + add_edge(file_nid, pipeline_nid, "contains", node) + return pipeline_nid + + def walk(node: Any, parent: str, *, in_steps: bool = False, current_stage: str | None = None, + in_agent: bool = False, current_function: str | None = None, + parallel_parent: bool = False) -> None: + nonlocal stage_number, step_number, parallel_number + name = call_name(node) + if node.type == "annotation": + annotation_name = node.child_by_field_name("name") + if ( + annotation_name is not None + and read(annotation_name).strip() == _SHARED_LIBRARY_ANNOTATION + ): + library_args = node.child_by_field_name("arguments") + library_ref = first_literal(library_args) if library_args is not None else None + if library_ref: + add_library(library_ref, node) + + if node.type == "function_definition": + function_name_node = node.child_by_field_name("name") + function_name = ( + read(function_name_node).strip() + if function_name_node is not None + else None + ) + function_nid = function_ids.get(function_name or "") + function_body = body(node) + if function_nid is not None and function_body is not None: + for child in function_body.named_children: + walk( + child, + function_nid, + in_steps=True, + current_stage=current_stage, + current_function=function_nid, + ) + return + + if name == "pipeline": + owner = add_pipeline(node) + pipeline_body = body(node) + if pipeline_body is not None: + for child in pipeline_body.named_children: + walk(child, owner, current_stage=current_stage) + return + + if name == "stage": + owner = parent if parallel_parent else (current_stage or parent) + stage_name = call_argument(node) or "stage" + stage_number += 1 + stage_nid = _make_id(stem, "stage", str(stage_number), stage_name) + add_node(stage_nid, stage_name, node, "jenkins_stage") + add_edge(owner, stage_nid, "contains", node) + stage_body = body(node) + if stage_body is not None: + for child in stage_body.named_children: + walk( + child, + stage_nid, + current_stage=stage_nid, + current_function=current_function, + ) + return + + if name in _PARALLEL_BLOCKS: + parallel_number += 1 + parallel_label = "JenkinsParallel" if name == "parallel" else "JenkinsMatrix" + parallel_nid = _make_id(stem, name, str(parallel_number)) + add_node(parallel_nid, parallel_label, node, f"jenkins_{name}") + add_edge(parent, parallel_nid, "contains", node) + + arguments = ( + node.child_by_field_name("arguments") + or node.child_by_field_name("args") + ) + if arguments is not None: + branch_number = 0 + for argument in arguments.named_children: + if argument.type != "map_item": + continue + branch_name_node = argument.child_by_field_name("key") + branch_name = ( + read(branch_name_node).strip() + if branch_name_node is not None + else f"branch-{branch_number + 1}" + ) + branch_number += 1 + branch_nid = _make_id(stem, name, str(parallel_number), "branch", branch_name) + add_node(branch_nid, branch_name, argument, f"jenkins_{name}_branch") + add_edge(parallel_nid, branch_nid, "contains", argument) + branch_body = argument.child_by_field_name("value") + if branch_body is not None: + for child in branch_body.named_children: + walk( + child, + branch_nid, + in_steps=True, + current_stage=current_stage, + current_function=current_function, + parallel_parent=True, + ) + + parallel_body = body(node) + if parallel_body is not None: + for child in parallel_body.named_children: + walk( + child, + parallel_nid, + current_stage=current_stage, + current_function=current_function, + parallel_parent=True, + ) + return + + if name == "steps": + steps_body = body(node) + if steps_body is not None: + for child in steps_body.named_children: + walk( + child, + parent, + in_steps=True, + current_stage=current_stage, + current_function=current_function, + ) + return + + if name in function_ids: + call_owner = current_function or parent + add_edge(call_owner, function_ids[name], "calls", node) + return + + if name in _IMAGE_CALLS and (not in_steps or name == "image"): + image_ref = call_argument(node) + if image_ref: + relation = "builds" if name == "docker.build" else "uses_image" + add_image(image_ref, parent, relation, node) + + next_parent = parent + if ( + in_steps + and name is not None + and name not in _STRUCTURAL_BLOCKS + and name not in _STAGE_CONTAINER_NAMES + ): + step_number += 1 + step_nid = _make_id(stem, "step", str(step_number), name) + add_node(step_nid, name, node, "jenkins_step") + add_edge(parent, step_nid, "contains", node) + next_parent = step_nid + if name in {"docker.build", "docker.image"}: + image_ref = call_argument(node) + if image_ref: + add_image( + image_ref, + step_nid, + "builds" if name == "docker.build" else "uses_image", + node, + ) + + if name == "agent": + in_agent = True + nested_body = body(node) + if nested_body is not None: + for child in nested_body.named_children: + walk( + child, + next_parent, + in_steps=in_steps, + current_stage=current_stage, + in_agent=in_agent, + current_function=current_function, + parallel_parent=parallel_parent, + ) + elif node.named_children: + for child in node.named_children: + walk( + child, + next_parent, + in_steps=in_steps, + current_stage=current_stage, + in_agent=in_agent, + current_function=current_function, + parallel_parent=parallel_parent, + ) + + # Declarative pipelines have a top-level pipeline call; scripted pipelines + # commonly start with node { ... }. Seed a pipeline either way so the + # resulting graph has one stable Jenkins root. + def unwrap_expression(node: Any) -> Any: + while node.type == "expression_statement" and len(node.named_children) == 1: + node = node.named_children[0] + return node + + top_level_calls = [unwrap_expression(child) for child in root.named_children] + + def collect_function_definitions(node: Any) -> None: + if node.type == "function_definition": + name_node = node.child_by_field_name("name") + if name_node is not None: + function_name = read(name_node).strip() + function_ids.setdefault(function_name, _make_id(stem, "function", function_name)) + function_nid = function_ids[function_name] + add_node(function_nid, function_name, node, "groovy_function") + add_edge(file_nid, function_nid, "contains", node) + for child in node.named_children: + collect_function_definitions(child) + + collect_function_definitions(root) + top_level_pipeline = next( + (child for child in top_level_calls if call_name(child) == "pipeline"), + None, + ) + owner = add_pipeline(top_level_pipeline or (top_level_calls[0] if top_level_calls else root)) + for child in root.named_children: + walk(child, owner, current_stage=None) + + return {"nodes": nodes, "edges": edges} diff --git a/tests/test_jenkins.py b/tests/test_jenkins.py new file mode 100644 index 000000000..4a051bdf1 --- /dev/null +++ b/tests/test_jenkins.py @@ -0,0 +1,122 @@ +from pathlib import Path + +from graphify.detect import FileType, classify_file +from graphify.extract import ( + _get_extractor, + collect_files, + extract_groovy, + extract_jenkinsfile, +) + + +JENKINSFILE = """\ +@Library('platform-shared') _ +def deployApp(String environment) { + sh "deploy ${environment}" + notifyTeam() +} +def notifyTeam() { + echo 'deployed' +} +pipeline { + agent { docker { image 'python:3.12' } } + stages { + stage('Build') { + steps { + sh 'make build' + checkout scm + docker.build('example/app:${BUILD_NUMBER}') + } + } + stage('Test') { + steps { + sh(script: 'pytest') + deployApp('prod') + } + } + stage('Fanout') { + parallel { + stage('Linux') { + steps { sh 'make linux' } + } + stage('Windows') { + steps { sh 'make windows' } + } + } + } + } +} +""" + + +def test_jenkinsfile_extracts_pipeline_stages_steps_and_images(tmp_path: Path): + path = tmp_path / "Jenkinsfile" + path.write_text(JENKINSFILE) + + result = extract_jenkinsfile(path) + + assert "error" not in result + by_label = {} + for node in result["nodes"]: + by_label.setdefault(node["label"], []).append(node) + + assert by_label["JenkinsPipeline"][0]["type"] == "jenkins_pipeline" + assert "Build" in by_label + assert "Test" in by_label + assert "sh" in by_label + assert "checkout" in by_label + assert "docker.build" in by_label + assert "python:3.12" in by_label + assert "example/app:${BUILD_NUMBER}" in by_label + assert "platform-shared" in by_label + assert by_label["deployApp"][0]["type"] == "groovy_function" + assert by_label["notifyTeam"][0]["type"] == "groovy_function" + assert "JenkinsParallel" in by_label + assert "Linux" in by_label + assert "Windows" in by_label + + relations = {(edge["relation"], edge["source"], edge["target"]) for edge in result["edges"]} + pipeline_id = by_label["JenkinsPipeline"][0]["id"] + build_id = by_label["Build"][0]["id"] + image_id = by_label["python:3.12"][0]["id"] + assert ("contains", pipeline_id, build_id) in relations + assert ("uses_image", pipeline_id, image_id) in relations + library_id = by_label["platform-shared"][0]["id"] + assert ("uses_library", pipeline_id, library_id) in relations + deploy_id = by_label["deployApp"][0]["id"] + notify_id = by_label["notifyTeam"][0]["id"] + parallel_id = by_label["JenkinsParallel"][0]["id"] + linux_id = by_label["Linux"][0]["id"] + windows_id = by_label["Windows"][0]["id"] + assert any( + relation == "calls" and target == deploy_id + for relation, _source, target in relations + ) + assert ("calls", deploy_id, notify_id) in relations + assert ("contains", by_label["Fanout"][0]["id"], parallel_id) in relations + assert ("contains", parallel_id, linux_id) in relations + assert ("contains", parallel_id, windows_id) in relations + + +def test_jenkinsfile_is_code_and_uses_special_extractor(tmp_path: Path): + path = tmp_path / "Jenkinsfile" + path.write_text("node { stage('Build') { sh 'make' } }\n") + + assert classify_file(path) is FileType.CODE + assert _get_extractor(path) is extract_jenkinsfile + + +def test_collect_files_includes_extensionless_jenkinsfile(tmp_path: Path): + jenkinsfile = tmp_path / "Jenkinsfile" + groovy = tmp_path / "build.groovy" + jenkinsfile.write_text("pipeline { agent any }\n") + groovy.write_text("class Build {}\n") + + assert collect_files(tmp_path) == sorted([jenkinsfile, groovy]) + + +def test_groovy_files_keep_the_generic_groovy_extractor(tmp_path: Path): + path = tmp_path / "build.groovy" + path.write_text("class Build {}\n") + + assert _get_extractor(path) is extract_groovy