test(benchmarks): add macro-heavy Jinja baseline

.github/workflows/cache-benchmark.yml

@@ -31,21 +31,30 @@ defaults:
 jobs:
   cache-benchmark-validation:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        profile: [small, macro-heavy]
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - name: Cache Rust
         uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
-      - name: Generate workload
-        run: python3 benchmarks/cache/scripts/generate_workload.py --self-check
-      - name: Validate workload
-        run: python3 benchmarks/cache/scripts/validate_workload.py benchmarks/cache/workloads/default
+      - name: Generate ${{ matrix.profile }} workload
+        run: >
+          python3 benchmarks/cache/scripts/generate_workload.py
+          --output benchmarks/cache/workloads/${{ matrix.profile }}
+          --profile ${{ matrix.profile }}
+          --self-check
+      - name: Validate ${{ matrix.profile }} workload
+        run: >
+          python3 benchmarks/cache/scripts/validate_workload.py
+          benchmarks/cache/workloads/${{ matrix.profile }}
       - name: Build dlin
         run: cargo build -p dlin --locked
-      - name: Validate cache behavior
+      - name: Validate ${{ matrix.profile }} cache behavior
         run: >
           python3 benchmarks/cache/scripts/run_benchmarks.py
-          --workload benchmarks/cache/workloads/default
+          --workload benchmarks/cache/workloads/${{ matrix.profile }}
           --binary target/debug/dlin
-          --results-dir benchmarks/cache/results/ci
+          --results-dir benchmarks/cache/results/ci/${{ matrix.profile }}
           --summary-file "$GITHUB_STEP_SUMMARY"
           --skip-timing

benchmarks/cache/README.md

@@ -31,10 +31,26 @@ python3 benchmarks/cache/scripts/generate_workload.py \
   --output benchmarks/cache/workloads/medium --profile medium --self-check
 ```
 
-The small profile has 64 models and medium has 512. Generation refuses to
-overwrite a non-empty directory. To replace a workload previously generated
-by this suite, pass `--force`; the marker file `workload_metadata.json` is
-required as a safety check:
+The small profile has 64 models and one macro file containing one macro, while
+medium has 512 models with the same macro setup. The `macro-heavy` profile
+keeps 64 models but includes 16 macro files containing 128 valid macros in the
+effective project macro prefix. Use it to measure both cumulative macro-source
+validation and parsing a larger prefix per model (twice per model in the
+current implementation):
+
+```sh
+python3 benchmarks/cache/scripts/generate_workload.py \
+  --output benchmarks/cache/workloads/macro-heavy --profile macro-heavy --self-check
+python3 benchmarks/cache/scripts/validate_workload.py \
+  benchmarks/cache/workloads/macro-heavy
+```
+
+Each model workload invokes the `benchmark_label` macro from `benchmark.sql`,
+so the larger profile exercises a real macro call as well as the additional
+valid macro definitions and source files.
+Generation refuses to overwrite a non-empty directory. To replace a workload
+previously generated by this suite, pass `--force`; the marker file
+`workload_metadata.json` is required as a safety check:
 
 ```sh
 python3 benchmarks/cache/scripts/generate_workload.py \

benchmarks/cache/scripts/generate_workload.py

@@ -12,7 +12,14 @@
 
 
 SUITE_ROOT = Path(__file__).resolve().parents[1]
-PROFILES = {"small": 64, "medium": 512}
+PROFILES = {
+    "small": {"model_count": 64, "macro_file_count": 1, "macro_count": 1},
+    "medium": {"model_count": 512, "macro_file_count": 1, "macro_count": 1},
+    # Keep the model count comparable to the default workload while making the
+    # effective macro prefix large enough to expose both per-model prefix
+    # parsing and build_macro_prefix's cumulative source validation.
+    "macro-heavy": {"model_count": 64, "macro_file_count": 16, "macro_count": 128},
+}
 
 
 def manifest_node(
@@ -38,7 +45,11 @@ def manifest_node(
     }
 
 
-def write_workload(root: Path, count: int) -> None:
+def write_workload(root: Path, profile: str) -> None:
+    profile_config = PROFILES[profile]
+    count = profile_config["model_count"]
+    macro_file_count = profile_config["macro_file_count"]
+    macro_count = profile_config["macro_count"]
     project = "cache_benchmark"
     sql_root = root / "sql_project"
     manifest_root = root / "manifest_project"
@@ -52,10 +63,22 @@ def write_workload(root: Path, count: int) -> None:
         "model-paths: [models]\n"
         "macro-paths: [macros]\n"
     )
-    macro = """{% macro benchmark_label(value) %}{{ value }}{% endmacro %}\n"""
+    macro_definitions = [
+        "{% macro benchmark_label(value) %}{{ value }}{% endmacro %}"
+    ]
+    macro_definitions.extend(
+        f"{{% macro benchmark_helper_{index:03d}(value) %}}"
+        "{{ value }}{% endmacro %}"
+        for index in range(1, macro_count)
+    )
     (sql_root / "dbt_project.yml").write_text(dbt_project, encoding="utf-8")
     (sql_root / "macros").mkdir(exist_ok=True)
-    (sql_root / "macros" / "benchmark.sql").write_text(macro, encoding="utf-8")
+    for file_index in range(macro_file_count):
+        start = macro_count * file_index // macro_file_count
+        end = macro_count * (file_index + 1) // macro_file_count
+        macro = "\n".join(macro_definitions[start:end]) + "\n"
+        filename = "benchmark.sql" if file_index == 0 else f"benchmark_{file_index:03d}.sql"
+        (sql_root / "macros" / filename).write_text(macro, encoding="utf-8")
     parent_name = "orders" if count == 1 else f"orders_{count - 2:04d}"
     (sql_root / "vars.yml").write_text(
         f"vars:\n  benchmark_parent: {parent_name}\n", encoding="utf-8"
@@ -125,10 +148,10 @@ def write_workload(root: Path, count: int) -> None:
     (manifest_root / "target" / "manifest.json").write_text(payload, encoding="utf-8")
 
     metadata = {
-        "profile": next(
-            (name for name, size in PROFILES.items() if size == count), "custom"
-        ),
+        "profile": profile,
         "model_count": count,
+        "macro_file_count": macro_file_count,
+        "macro_count": macro_count,
     }
     metadata["files"] = {}
     for file in sorted(root.rglob("*")):
@@ -168,8 +191,8 @@ def main() -> int:
     args = parser.parse_args()
     if args.self_check:
         with tempfile.TemporaryDirectory() as first, tempfile.TemporaryDirectory() as second:
-            write_workload(Path(first), PROFILES[args.profile])
-            write_workload(Path(second), PROFILES[args.profile])
+            write_workload(Path(first), args.profile)
+            write_workload(Path(second), args.profile)
             if tree_digest(Path(first)) != tree_digest(Path(second)):
                 parser.error("workload generation is not deterministic")
     args.output = args.output.resolve()
@@ -181,8 +204,15 @@ def main() -> int:
                 parser.error("output must be empty; use --force only for a previously generated workload")
             shutil.rmtree(args.output)
     args.output.mkdir(parents=True, exist_ok=True)
-    write_workload(args.output, PROFILES[args.profile])
-    print(f"generated {args.profile} workload ({PROFILES[args.profile]} models) at {args.output}")
+    write_workload(args.output, args.profile)
+    config = PROFILES[args.profile]
+    print(
+        f"generated {args.profile} workload "
+        f"({config['model_count']} models, "
+        f"macro_files={config['macro_file_count']}, "
+        f"macros={config['macro_count']}) "
+        f"at {args.output}"
+    )
     return 0
 
 

benchmarks/cache/scripts/run_benchmarks.py

@@ -308,7 +308,10 @@ def validation_summary(metadata: dict[str, object]) -> str:
     lines = [
         "## dlin cache semantic validation",
         "",
-        f"- Fixture: `{workload_metadata['profile']}` ({workload_metadata['model_count']} models)",
+        f"- Fixture: `{workload_metadata['profile']}` "
+        f"({workload_metadata['model_count']} models, "
+        f"macro_files={workload_metadata['macro_file_count']}, "
+        f"macros={workload_metadata['macro_count']})",
         f"- dlin: `{metadata['binary_version']}`",
         f"- Timing values: omitted (timing run: {'yes' if metadata['timing'] else 'no'})",
         "",

benchmarks/cache/scripts/validate_workload.py

@@ -27,6 +27,16 @@ def main() -> int:
         or metadata["model_count"] < 1
     ):
         parser.error("workload_metadata.json has an invalid model_count")
+    if (
+        not isinstance(metadata.get("macro_count"), int)
+        or metadata["macro_count"] < 1
+    ):
+        parser.error("workload_metadata.json has an invalid macro_count")
+    if (
+        not isinstance(metadata.get("macro_file_count"), int)
+        or metadata["macro_file_count"] < 1
+    ):
+        parser.error("workload_metadata.json has an invalid macro_file_count")
     expected = metadata.get("files", {})
     actual = {}
     for path in sorted(root.rglob("*")):
@@ -43,6 +53,23 @@ def main() -> int:
         parser.error(f"manifest is missing nodes or sources: {manifest_path}")
     if (root / "sql_project/target/manifest.json").exists():
         parser.error("SQL workload must not contain a target/manifest.json")
+    macro_files = sorted((root / "sql_project/macros").glob("*.sql"))
+    if (
+        len(macro_files) != metadata["macro_file_count"]
+        or macro_files[0].name != "benchmark.sql"
+    ):
+        parser.error("SQL workload macro files differ from workload_metadata.json")
+    macro_source = "\n".join(
+        path.read_text(encoding="utf-8") for path in macro_files
+    )
+    macro_definitions = macro_source.count("{% macro ")
+    if macro_definitions != metadata["macro_count"]:
+        parser.error(
+            "macro definition count differs from workload_metadata.json: "
+            f"{macro_definitions} != {metadata['macro_count']}"
+        )
+    if "{% macro benchmark_label(" not in macro_source:
+        parser.error("SQL workload is missing the invoked benchmark_label macro")
     manifest_files = sorted(
         path.relative_to(root / "manifest_project").as_posix()
         for path in (root / "manifest_project").rglob("*")
@@ -66,7 +93,12 @@ def main() -> int:
         "{{ ref(" in path.read_text(encoding="utf-8") for path in models
     ):
         parser.error("SQL workload does not exercise ref() extraction")
-    print(f"validated {metadata['profile']} workload ({metadata['model_count']} models)")
+    print(
+        f"validated {metadata['profile']} workload "
+        f"({metadata['model_count']} models, "
+        f"macro_files={metadata['macro_file_count']}, "
+        f"macros={metadata['macro_count']})"
+    )
     return 0