[FEATURE] Ship agent-skill guidance for configuring data sources and expectations

.github/workflows/ci.yml

@@ -949,6 +949,9 @@ jobs:
       - name: Import Great Expectations
         run: python -c "import great_expectations as gx; print('Successfully imported GX Version:', gx.__version__)"
 
+      - name: Check installed agent skills and schema catalogs
+        run: python ci/checks/check_installed_agent_skills.py
+
   ci-required:
     # Single required status check for branch protection.
     # Passes when every upstream job either succeeded or was skipped

ci/checks/check_installed_agent_skills.py

@@ -0,0 +1,206 @@
+"""
+Purpose: guard the installed package against silently dropping the bundled agent
+skills or their schema catalogs.
+
+The skills and the two catalog indexes they depend on all ship through
+`package_data` glob patterns rather than through code, which means nothing enforces
+that the patterns stay in sync with what actually lives in the source tree: a pattern
+narrowed by an unrelated edit, or a file added under a directory the patterns do not
+reach, fails silently. `pip install .` and `import great_expectations` both succeed
+either way, so nothing in the ordinary import-and-run check catches it.
+
+This script is meant to run after `pip install .`, against the resulting installed
+package, and checks the properties a user actually depends on:
+
+* both bundled skills resolve the way an installed package resolves them -- through
+  the import system, not by checking that some files happen to exist;
+* both schema catalog indexes are present, and each schema tree ships more than just
+  its index;
+* the `skills list` subcommand names both skills;
+* installing from the running package produces content that matches its own
+  ownership manifest;
+* every file the source tree bundles for a skill actually made it into the installed
+  package, not just the files that happen to make the skill resolve.
+
+Run directly with `python ci/checks/check_installed_agent_skills.py` from the
+repository root, with the package already installed in the active environment.
+"""
+
+from __future__ import annotations
+
+import io
+import sys
+import tempfile
+from contextlib import redirect_stdout
+from pathlib import Path
+from typing import Final
+
+import great_expectations
+from great_expectations import __main__ as command_line
+from great_expectations.agent_skills import installer
+from great_expectations.agent_skills.installer import (
+    SkillTarget,
+    install_skills,
+    iter_bundled_skills,
+    read_skill_manifest,
+)
+
+#: The skills this package currently bundles, named explicitly rather than merely
+#: counted -- a rename or a dropped skill is then reported by name instead of as an
+#: unexplained count mismatch.
+EXPECTED_SKILLS: Final = frozenset({"gx-configure-data-source", "gx-configure-expectations"})
+
+#: The schema trees the package ships alongside the skills, each carrying its own
+#: catalog index, relative to the installed package root.
+SCHEMA_TREES: Final = (
+    Path("expectations", "core", "schemas"),
+    Path("datasource", "fluent", "schemas"),
+)
+
+INDEX_NAME: Final = "index.json"
+
+
+def check_bundled_skills_resolve() -> list[Path]:
+    """Both skills must be found the way an installed package is found: through the
+    import system, not by checking that some files happen to exist.
+
+    ``iter_bundled_skills`` is what every install and list run relies on to locate the
+    skills, and it is also what raises when a packaging pattern ships some of a
+    skill's files and drops others -- the shape a too-narrow glob produces. Calling it
+    here, rather than checking paths by hand, is what makes this a check on skill
+    *resolution*.
+    """
+    skills = sorted(iter_bundled_skills(), key=lambda skill: skill.name)
+    names = {skill.name for skill in skills}
+    assert names == EXPECTED_SKILLS, (
+        f"expected the installed package to bundle {sorted(EXPECTED_SKILLS)}, found {sorted(names)}"
+    )
+    for skill in skills:
+        assert (skill / "SKILL.md").is_file(), f"{skill} has no SKILL.md"
+    return skills
+
+
+def check_catalog_indexes(installed_root: Path) -> None:
+    """Both catalog indexes must ship at their documented location."""
+    for tree in SCHEMA_TREES:
+        index = installed_root / tree / INDEX_NAME
+        assert index.is_file(), f"{index} was not found in the installed package"
+
+
+def check_schema_counts_nonzero(installed_root: Path) -> dict[str, int]:
+    """Each schema tree must ship more than just its index."""
+    counts: dict[str, int] = {}
+    for tree in SCHEMA_TREES:
+        directory = installed_root / tree
+        count = sum(1 for path in directory.rglob("*.json") if path.name != INDEX_NAME)
+        assert count > 0, f"no schema JSON files were found under {directory}"
+        counts[tree.as_posix()] = count
+    return counts
+
+
+def check_skills_list_names_both(project_root: Path) -> str:
+    """The ``skills list`` subcommand must name both skills, run the way a user runs it.
+
+    Invoked in-process through the same entry point ``python -m great_expectations``
+    calls, rather than shelled out to, so the exact code path a user runs is exercised
+    without depending on how the interpreter running this script happens to be found.
+    """
+    buffer = io.StringIO()
+    with redirect_stdout(buffer):
+        exit_code = command_line.main(["skills", "list", "--project-root", str(project_root)])
+    output = buffer.getvalue()
+    assert exit_code == 0, f"'skills list' exited {exit_code}:\n{output}"
+    for name in EXPECTED_SKILLS:
+        assert name in output, f"'skills list' did not mention {name}:\n{output}"
+    return output
+
+
+def check_installed_digests_match_manifest(project_root: Path) -> None:
+    """Installing from the running package must produce content matching its own
+    ownership manifest.
+
+    That match is what lets a later run tell an untouched install apart from one the
+    user edited. Checking it here, against a genuinely packaged and installed
+    distribution, covers a case the test suite cannot reach: the suite exercises a
+    fixture tree or a source checkout under an editable install, neither of which is
+    the artifact a user actually receives.
+
+    Hashing is done with the installer's own ``_tree_digest`` rather than a second,
+    independently written function: the manifest's ``content_sha256`` field is defined
+    as that function's output, so the only way to ask "does this destination still
+    match what its manifest recorded" is to recompute the same function and compare --
+    a differently framed hash would disagree with the manifest even for byte-identical
+    content, and this check would fail on every run rather than only on a real
+    regression. That is not circular, because the two hashes are taken over different
+    trees: the manifest's value is computed from the installed package's own bundled
+    directory, while this recomputes over the copy placed in the project. Equality is
+    therefore a real property of the install pipeline -- that ``shutil.copytree``
+    reproduced the source directory byte for byte.
+    """
+    report = install_skills(project_root, targets=(SkillTarget.AGENTS, SkillTarget.CLAUDE))
+    assert not report.failed, (
+        f"installing into a scratch project reported failures: {report.failed}"
+    )
+    assert report.installed, "installing into a scratch project installed nothing"
+    for destination in report.installed:
+        manifest = read_skill_manifest(destination)
+        assert manifest is not None, f"{destination} has no ownership manifest after install"
+        recorded = manifest.get("content_sha256")
+        actual = installer._tree_digest(destination)
+        assert actual == recorded, (
+            f"{destination} hashes to {actual}, but its manifest records {recorded} -- the "
+            "installed content does not match what was written down for it"
+        )
+
+
+def check_every_bundled_file_shipped(source_root: Path, installed_root: Path) -> None:
+    """Every file under the source skills tree must exist at the same relative path in
+    the installed package.
+
+    Every other check here can pass while a packaging pattern still drops a file that
+    is neither a ``SKILL.md`` nor a markdown reference -- a script or an image added to
+    a skill directory, say -- because nothing else compares the two trees file for
+    file. ``iter_bundled_skills`` would not notice: it only requires ``SKILL.md``.
+    """
+    missing = [
+        path.relative_to(source_root).as_posix()
+        for path in sorted(source_root.rglob("*"))
+        if path.is_file()
+        and not path.is_symlink()
+        and not (installed_root / path.relative_to(source_root)).is_file()
+    ]
+    assert not missing, (
+        f"these files exist under {source_root} but were not found in the installed package "
+        f"at {installed_root} -- check the packaging patterns for the skills tree: {missing}"
+    )
+
+
+def main() -> None:
+    installed_root = Path(great_expectations.__file__).resolve().parent
+    repo_root = Path(__file__).resolve().parents[2]
+    source_skills_root = repo_root / "great_expectations" / ".agents" / "skills"
+
+    try:
+        skills = check_bundled_skills_resolve()
+        check_catalog_indexes(installed_root)
+        counts = check_schema_counts_nonzero(installed_root)
+
+        with tempfile.TemporaryDirectory(prefix="gx-installed-skills-guard-") as scratch:
+            project_root = Path(scratch)
+            check_skills_list_names_both(project_root)
+            check_installed_digests_match_manifest(project_root)
+
+        check_every_bundled_file_shipped(source_skills_root, skills[0].parent)
+    except (AssertionError, OSError) as error:
+        print(f"[ERROR] {error}")
+        sys.exit(1)
+
+    schema_summary = ", ".join(f"{count} under {tree}" for tree, count in counts.items())
+    print(
+        f"Installed agent skills are complete: {len(skills)} skills "
+        f"({', '.join(sorted(skill.name for skill in skills))}), schemas {schema_summary}."
+    )
+
+
+if __name__ == "__main__":
+    main()

docs/adr/0006-ship-agent-skills-with-the-package.md

@@ -0,0 +1,134 @@
+# 6. Ship agent skills with the package
+
+Date: 2026-08-13
+
+## Status
+
+Accepted
+
+## Context
+
+Data practitioners increasingly configure and validate data through a coding
+agent rather than by writing every line of Python themselves. An agent's
+general programming knowledge does not tell it the current, correct sequence
+of calls for a specific library: which factory method to call for a given
+connection type, in what order a validation suite has to be registered before
+expectations are added to it, or how to handle a secret without ever printing
+it to the conversation. Left to infer this from the source or from
+out-of-date training data, an agent produces plausible-looking code that is
+subtly wrong at least as often as it produces working code, and a user who
+does not already know the right pattern has no way to tell the two apart.
+
+Closing that gap requires guidance that an agent can actually find and use.
+That means it has to live where an agent's tooling already looks, in a form
+the agent's platform already knows how to read, and it has to stay accurate
+for whatever version of the library the user has installed — guidance
+written against an API that has since changed is worse than no guidance,
+because it is confidently wrong instead of visibly absent.
+
+## Decision
+
+We ship a set of "skills" — self-contained guidance documents for a coding
+agent — as part of the `great_expectations` distribution, and give users a
+command to place them where their agent looks for them.
+
+**Format.** Each skill is a directory containing one entry document, plus
+supporting reference material one directory level below it. This is an open
+format, not something specific to Great Expectations: multiple coding-agent
+tools already read directories shaped this way, so publishing skills in this
+form makes them usable by every agent whose platform speaks the format,
+without our writing a separate integration per agent. A proprietary or
+single-vendor shape would have bought nothing for the additional maintenance
+of yet another format, and would have worked with only one agent.
+
+**Location.** The skill content lives inside the installed package itself,
+not behind a URL the agent fetches at runtime and not something generated on
+demand. The reason is version matching: the correct guidance for calling a
+fluent factory method or registering a suite is a function of the exact
+`great_expectations` release installed, and an install of the package is the
+one artifact guaranteed to carry the version the guidance has to match. A
+separately hosted copy can drift out of sync with any given install the
+moment either one changes independently, silently handing an agent
+instructions for an API surface that no longer matches what is on disk.
+Shipping the content in the package ties its version to the code's version by
+construction, so the normal act of installing or upgrading the package is
+also what keeps the guidance current.
+
+**Command surface.** The command to place the bundled skills into a project
+is invoked as a module, `python -m great_expectations …`, rather than through
+a new console-script entry point installed onto the user's `PATH`. Great
+Expectations previously shipped a console-script command-line interface and
+removed it. Reintroducing one — even a minimal one — brings back the
+packaging-level machinery a console script requires and the platform-specific
+quirks of `PATH`-installed executables (name collisions, `PATH` not being set
+up in every environment a Python package is used from, different behavior
+across virtual environments and editable installs), to serve what is, in
+substance, an occasional local file-management step for a library that is
+not a command-line application. `python -m` needs none of that: it uses the
+same import machinery already required to use the library at all, so it
+behaves identically in every environment where `import great_expectations`
+already works.
+
+**Install model.** The command places the bundled skills, by copying or on
+request by linking, into the discovery directories a project's coding agent
+reads — `.agents/skills` for Codex and Cursor, `.claude/skills` for Claude
+Code and Cursor — alongside a small manifest recording what was installed and
+a hash of its content. Several principles follow from treating the
+destination as belonging to the user, not to the package:
+
+- A destination that already holds exactly what would be installed is left
+  completely alone. The manifest is what lets a repeat run tell "nothing to
+  do" apart from "something changed" without guessing from the file contents
+  alone, which is what makes the command safe to run again after every
+  upgrade, or simply on the suspicion that it was never run at all.
+- The tool never silently overwrites something it did not create. A
+  directory with no record of having been installed by this package is left
+  alone unconditionally — there is no option that overwrites it — because
+  nothing in a directory it never wrote can be told apart from a user's own
+  work.
+- Once a directory carries that record, the tool can tell its own untouched
+  copy apart from one the user has since edited, and refuses to replace the
+  latter without an explicit override. A command meant to be safe to run
+  again after every upgrade cannot also be a command that discards local
+  edits as a side effect of checking for updates.
+- An upgrade builds the replacement in full beside the destination and moves
+  it into place, never rewriting files where they sit. A process that dies
+  partway through therefore never leaves behind a skill with some files at the
+  new version and some at the old — an agent reading a directory in that state
+  would follow guidance that no single release ever actually shipped, which is
+  worse than the outdated version it was replacing.
+
+## Consequences
+
+An agent whose platform reads this open format gets accurate, version-matched
+guidance the moment the package is installed and the install command is run,
+with no bespoke integration effort on our part and none required of the
+agent's maintainers. The same content is available to any other tool that
+scans installed packages for it, at no additional cost, because it sits at a
+predictable path inside the package rather than behind custom retrieval
+logic.
+
+The guidance now has to be kept in step with the fluent API it describes, the
+same way any other part of the package does, or it degrades into the exact
+failure mode — instructions for an API that no longer matches what is
+installed — that shipping it in-package was meant to prevent.
+
+The install copies by default, so a package upgrade alone does not update
+guidance already placed in a project; the install command has to be run again
+to pick up a new version. That default exists because copying is the only
+form every platform this content runs on is known to treat the same way a
+real directory is treated, and the only one that survives the package being
+upgraded or removed. Re-running the install command is the price of that
+reliability, and it is cheap precisely because re-running it is always safe.
+Linking directly to the package's own copy is available for users who want
+guidance that tracks the installed version without re-running anything;
+choosing it accepts, in exchange, that a project's guidance can change
+without an explicit action, and that on a platform that will not create
+links at all it is reported as a failure rather than falling back silently
+to a copy.
+
+There is no globally installed executable to remember; the command is only
+reachable through `python -m`, which requires knowing the package is
+installed in the environment being used — a smaller surface than a
+console script, and one that trades a small amount of discoverability for
+never depending on how a user's `PATH` happens to be configured.