From 6c2abcaa0f06f09e084fa5ea4f7492b07d31005b Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sat, 29 Aug 2026 15:16:19 +0000 Subject: [PATCH 1/2] test(benchmarks): add macro-heavy Jinja workload --- benchmarks/cache/README.md | 24 +++++++-- benchmarks/cache/scripts/generate_workload.py | 52 +++++++++++++++---- benchmarks/cache/scripts/run_benchmarks.py | 5 +- benchmarks/cache/scripts/validate_workload.py | 34 +++++++++++- 4 files changed, 98 insertions(+), 17 deletions(-) diff --git a/benchmarks/cache/README.md b/benchmarks/cache/README.md index 8b7b664..bd33f3a 100644 --- a/benchmarks/cache/README.md +++ b/benchmarks/cache/README.md @@ -31,10 +31,26 @@ python3 benchmarks/cache/scripts/generate_workload.py \ --output benchmarks/cache/workloads/medium --profile medium --self-check ``` -The small profile has 64 models and medium has 512. Generation refuses to -overwrite a non-empty directory. To replace a workload previously generated -by this suite, pass `--force`; the marker file `workload_metadata.json` is -required as a safety check: +The small profile has 64 models and one macro file containing one macro, while +medium has 512 models with the same macro setup. The `macro-heavy` profile +keeps 64 models but includes 16 macro files containing 128 valid macros in the +effective project macro prefix. Use it to measure both cumulative macro-source +validation and parsing a larger prefix per model (twice per model in the +current implementation): + +```sh +python3 benchmarks/cache/scripts/generate_workload.py \ + --output benchmarks/cache/workloads/macro-heavy --profile macro-heavy --self-check +python3 benchmarks/cache/scripts/validate_workload.py \ + benchmarks/cache/workloads/macro-heavy +``` + +Each model workload invokes the `benchmark_label` macro from `benchmark.sql`, +so the larger profile exercises a real macro call as well as the additional +valid macro definitions and source files. +Generation refuses to overwrite a non-empty directory. To replace a workload +previously generated by this suite, pass `--force`; the marker file +`workload_metadata.json` is required as a safety check: ```sh python3 benchmarks/cache/scripts/generate_workload.py \ diff --git a/benchmarks/cache/scripts/generate_workload.py b/benchmarks/cache/scripts/generate_workload.py index 98d4de5..751314a 100644 --- a/benchmarks/cache/scripts/generate_workload.py +++ b/benchmarks/cache/scripts/generate_workload.py @@ -12,7 +12,14 @@ SUITE_ROOT = Path(__file__).resolve().parents[1] -PROFILES = {"small": 64, "medium": 512} +PROFILES = { + "small": {"model_count": 64, "macro_file_count": 1, "macro_count": 1}, + "medium": {"model_count": 512, "macro_file_count": 1, "macro_count": 1}, + # Keep the model count comparable to the default workload while making the + # effective macro prefix large enough to expose both per-model prefix + # parsing and build_macro_prefix's cumulative source validation. + "macro-heavy": {"model_count": 64, "macro_file_count": 16, "macro_count": 128}, +} def manifest_node( @@ -38,7 +45,11 @@ def manifest_node( } -def write_workload(root: Path, count: int) -> None: +def write_workload(root: Path, profile: str) -> None: + profile_config = PROFILES[profile] + count = profile_config["model_count"] + macro_file_count = profile_config["macro_file_count"] + macro_count = profile_config["macro_count"] project = "cache_benchmark" sql_root = root / "sql_project" manifest_root = root / "manifest_project" @@ -52,10 +63,22 @@ def write_workload(root: Path, count: int) -> None: "model-paths: [models]\n" "macro-paths: [macros]\n" ) - macro = """{% macro benchmark_label(value) %}{{ value }}{% endmacro %}\n""" + macro_definitions = [ + "{% macro benchmark_label(value) %}{{ value }}{% endmacro %}" + ] + macro_definitions.extend( + f"{{% macro benchmark_helper_{index:03d}(value) %}}" + "{{ value }}{% endmacro %}" + for index in range(1, macro_count) + ) (sql_root / "dbt_project.yml").write_text(dbt_project, encoding="utf-8") (sql_root / "macros").mkdir(exist_ok=True) - (sql_root / "macros" / "benchmark.sql").write_text(macro, encoding="utf-8") + for file_index in range(macro_file_count): + start = macro_count * file_index // macro_file_count + end = macro_count * (file_index + 1) // macro_file_count + macro = "\n".join(macro_definitions[start:end]) + "\n" + filename = "benchmark.sql" if file_index == 0 else f"benchmark_{file_index:03d}.sql" + (sql_root / "macros" / filename).write_text(macro, encoding="utf-8") parent_name = "orders" if count == 1 else f"orders_{count - 2:04d}" (sql_root / "vars.yml").write_text( f"vars:\n benchmark_parent: {parent_name}\n", encoding="utf-8" @@ -125,10 +148,10 @@ def write_workload(root: Path, count: int) -> None: (manifest_root / "target" / "manifest.json").write_text(payload, encoding="utf-8") metadata = { - "profile": next( - (name for name, size in PROFILES.items() if size == count), "custom" - ), + "profile": profile, "model_count": count, + "macro_file_count": macro_file_count, + "macro_count": macro_count, } metadata["files"] = {} for file in sorted(root.rglob("*")): @@ -168,8 +191,8 @@ def main() -> int: args = parser.parse_args() if args.self_check: with tempfile.TemporaryDirectory() as first, tempfile.TemporaryDirectory() as second: - write_workload(Path(first), PROFILES[args.profile]) - write_workload(Path(second), PROFILES[args.profile]) + write_workload(Path(first), args.profile) + write_workload(Path(second), args.profile) if tree_digest(Path(first)) != tree_digest(Path(second)): parser.error("workload generation is not deterministic") args.output = args.output.resolve() @@ -181,8 +204,15 @@ def main() -> int: parser.error("output must be empty; use --force only for a previously generated workload") shutil.rmtree(args.output) args.output.mkdir(parents=True, exist_ok=True) - write_workload(args.output, PROFILES[args.profile]) - print(f"generated {args.profile} workload ({PROFILES[args.profile]} models) at {args.output}") + write_workload(args.output, args.profile) + config = PROFILES[args.profile] + print( + f"generated {args.profile} workload " + f"({config['model_count']} models, " + f"macro_files={config['macro_file_count']}, " + f"macros={config['macro_count']}) " + f"at {args.output}" + ) return 0 diff --git a/benchmarks/cache/scripts/run_benchmarks.py b/benchmarks/cache/scripts/run_benchmarks.py index 336815b..9b6444b 100644 --- a/benchmarks/cache/scripts/run_benchmarks.py +++ b/benchmarks/cache/scripts/run_benchmarks.py @@ -308,7 +308,10 @@ def validation_summary(metadata: dict[str, object]) -> str: lines = [ "## dlin cache semantic validation", "", - f"- Fixture: `{workload_metadata['profile']}` ({workload_metadata['model_count']} models)", + f"- Fixture: `{workload_metadata['profile']}` " + f"({workload_metadata['model_count']} models, " + f"macro_files={workload_metadata['macro_file_count']}, " + f"macros={workload_metadata['macro_count']})", f"- dlin: `{metadata['binary_version']}`", f"- Timing values: omitted (timing run: {'yes' if metadata['timing'] else 'no'})", "", diff --git a/benchmarks/cache/scripts/validate_workload.py b/benchmarks/cache/scripts/validate_workload.py index 62a6fbc..237f576 100644 --- a/benchmarks/cache/scripts/validate_workload.py +++ b/benchmarks/cache/scripts/validate_workload.py @@ -27,6 +27,16 @@ def main() -> int: or metadata["model_count"] < 1 ): parser.error("workload_metadata.json has an invalid model_count") + if ( + not isinstance(metadata.get("macro_count"), int) + or metadata["macro_count"] < 1 + ): + parser.error("workload_metadata.json has an invalid macro_count") + if ( + not isinstance(metadata.get("macro_file_count"), int) + or metadata["macro_file_count"] < 1 + ): + parser.error("workload_metadata.json has an invalid macro_file_count") expected = metadata.get("files", {}) actual = {} for path in sorted(root.rglob("*")): @@ -43,6 +53,23 @@ def main() -> int: parser.error(f"manifest is missing nodes or sources: {manifest_path}") if (root / "sql_project/target/manifest.json").exists(): parser.error("SQL workload must not contain a target/manifest.json") + macro_files = sorted((root / "sql_project/macros").glob("*.sql")) + if ( + len(macro_files) != metadata["macro_file_count"] + or macro_files[0].name != "benchmark.sql" + ): + parser.error("SQL workload macro files differ from workload_metadata.json") + macro_source = "\n".join( + path.read_text(encoding="utf-8") for path in macro_files + ) + macro_definitions = macro_source.count("{% macro ") + if macro_definitions != metadata["macro_count"]: + parser.error( + "macro definition count differs from workload_metadata.json: " + f"{macro_definitions} != {metadata['macro_count']}" + ) + if "{% macro benchmark_label(" not in macro_source: + parser.error("SQL workload is missing the invoked benchmark_label macro") manifest_files = sorted( path.relative_to(root / "manifest_project").as_posix() for path in (root / "manifest_project").rglob("*") @@ -66,7 +93,12 @@ def main() -> int: "{{ ref(" in path.read_text(encoding="utf-8") for path in models ): parser.error("SQL workload does not exercise ref() extraction") - print(f"validated {metadata['profile']} workload ({metadata['model_count']} models)") + print( + f"validated {metadata['profile']} workload " + f"({metadata['model_count']} models, " + f"macro_files={metadata['macro_file_count']}, " + f"macros={metadata['macro_count']})" + ) return 0 From ab154be662e1d9b38d72e8f190db5934690f91e1 Mon Sep 17 00:00:00 2001 From: eitsupi Date: Sat, 29 Aug 2026 15:22:32 +0000 Subject: [PATCH 2/2] ci: validate macro-heavy cache workload --- .github/workflows/cache-benchmark.yml | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/.github/workflows/cache-benchmark.yml b/.github/workflows/cache-benchmark.yml index 8818742..51923f5 100644 --- a/.github/workflows/cache-benchmark.yml +++ b/.github/workflows/cache-benchmark.yml @@ -31,21 +31,30 @@ defaults: jobs: cache-benchmark-validation: runs-on: ubuntu-latest + strategy: + matrix: + profile: [small, macro-heavy] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Cache Rust uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - - name: Generate workload - run: python3 benchmarks/cache/scripts/generate_workload.py --self-check - - name: Validate workload - run: python3 benchmarks/cache/scripts/validate_workload.py benchmarks/cache/workloads/default + - name: Generate ${{ matrix.profile }} workload + run: > + python3 benchmarks/cache/scripts/generate_workload.py + --output benchmarks/cache/workloads/${{ matrix.profile }} + --profile ${{ matrix.profile }} + --self-check + - name: Validate ${{ matrix.profile }} workload + run: > + python3 benchmarks/cache/scripts/validate_workload.py + benchmarks/cache/workloads/${{ matrix.profile }} - name: Build dlin run: cargo build -p dlin --locked - - name: Validate cache behavior + - name: Validate ${{ matrix.profile }} cache behavior run: > python3 benchmarks/cache/scripts/run_benchmarks.py - --workload benchmarks/cache/workloads/default + --workload benchmarks/cache/workloads/${{ matrix.profile }} --binary target/debug/dlin - --results-dir benchmarks/cache/results/ci + --results-dir benchmarks/cache/results/ci/${{ matrix.profile }} --summary-file "$GITHUB_STEP_SUMMARY" --skip-timing