diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d6de87..a20d6ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Change Log +## 2.3.0 + +### Framework + +- **Force extra dbt model dependencies with `depends_on`.** On any `.model.json`, set `"depends_on": ["processed_orders", ...]` to emit `--depends_on: {{ ref('...') }}` comments in generated SQL so dbt can discover refs that are otherwise hidden at parse time (for example inside `{% if execute %}`). + ## 2.2.1 ### Security diff --git a/docs/models/README.md b/docs/models/README.md index f939b81..73bea2e 100644 --- a/docs/models/README.md +++ b/docs/models/README.md @@ -417,6 +417,23 @@ Understanding the relationship between model types: - **Intermediate models** feed **mart models** - **Mart models** feed **BI tools and dashboards** +### Forcing dbt dependencies + +dbt builds its DAG from `ref()` calls it can see at parse time. When a dependency is only used inside a branch that is skipped during parse (for example `{% if execute %}`), add the upstream model names under `depends_on` on the `.model.json`. DJ emits one SQL comment per value: + +```jsonc +{ + "depends_on": ["processed_orders", "another_model"] +} +``` + +```sql +--depends_on: {{ ref('processed_orders') }} +--depends_on: {{ ref('another_model') }} +``` + +See [dbt: Forcing dependencies](https://docs.getdbt.com/reference/dbt-jinja-functions/ref?version=2#forcing-dependencies). + ### Performance Optimization - **Materialization**: Choose appropriate strategy for data volume diff --git a/package-lock.json b/package-lock.json index 73b8584..a50e9d3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "dj", - "version": "2.2.1", + "version": "2.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "dj", - "version": "2.2.1", + "version": "2.3.0", "license": "Apache-2.0", "workspaces": [ "web" diff --git a/package.json b/package.json index d51434b..f5cd7a0 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "type": "git", "url": "https://github.com/Workday/dj.git" }, - "version": "2.2.1", + "version": "2.3.0", "workspaces": [ "web" ], diff --git a/schemas/model.depends_on.schema.json b/schemas/model.depends_on.schema.json new file mode 100644 index 0000000..1ca5d93 --- /dev/null +++ b/schemas/model.depends_on.schema.json @@ -0,0 +1,11 @@ +{ + "$id": "model.depends_on.schema.json", + "title": "Schema Model Depends On", + "description": "Additional model names whose ref() edges should be forced in generated SQL via --depends_on comments (for refs hidden from dbt parse, e.g. inside {% if execute %})", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "model.ref.schema.json" + } +} diff --git a/schemas/model.type.int_join_column.schema.json b/schemas/model.type.int_join_column.schema.json index 23b62fe..f2f11c0 100644 --- a/schemas/model.type.int_join_column.schema.json +++ b/schemas/model.type.int_join_column.schema.json @@ -30,6 +30,9 @@ "meta": { "$ref": "model.meta.schema.json" }, + "depends_on": { + "$ref": "model.depends_on.schema.json" + }, "data_tests": { "$ref": "model.data_tests.schema.json" }, diff --git a/schemas/model.type.int_join_models.schema.json b/schemas/model.type.int_join_models.schema.json index ff760f8..bdcb6c7 100644 --- a/schemas/model.type.int_join_models.schema.json +++ b/schemas/model.type.int_join_models.schema.json @@ -14,6 +14,7 @@ "lightdash": { "$ref": "model.lightdash.schema.json" }, "tags": { "$ref": "model.tags.schema.json" }, "meta": { "$ref": "model.meta.schema.json" }, + "depends_on": { "$ref": "model.depends_on.schema.json" }, "data_tests": { "$ref": "model.data_tests.schema.json" }, "materialization": { "$ref": "model.materialization.schema.json" }, "materialized": { "$ref": "model.materialized.schema.json" }, diff --git a/schemas/model.type.int_lookback_model.schema.json b/schemas/model.type.int_lookback_model.schema.json index c5f1366..9ceefac 100644 --- a/schemas/model.type.int_lookback_model.schema.json +++ b/schemas/model.type.int_lookback_model.schema.json @@ -14,6 +14,7 @@ "lightdash": { "$ref": "model.lightdash.schema.json" }, "tags": { "$ref": "model.tags.schema.json" }, "meta": { "$ref": "model.meta.schema.json" }, + "depends_on": { "$ref": "model.depends_on.schema.json" }, "data_tests": { "$ref": "model.data_tests.schema.json" }, "materialization": { "$ref": "model.materialization.schema.json" }, "materialized": { "$ref": "model.materialized.schema.json" }, diff --git a/schemas/model.type.int_rollup_model.schema.json b/schemas/model.type.int_rollup_model.schema.json index 52f3a53..2778828 100644 --- a/schemas/model.type.int_rollup_model.schema.json +++ b/schemas/model.type.int_rollup_model.schema.json @@ -14,6 +14,7 @@ "lightdash": { "$ref": "model.lightdash.schema.json" }, "tags": { "$ref": "model.tags.schema.json" }, "meta": { "$ref": "model.meta.schema.json" }, + "depends_on": { "$ref": "model.depends_on.schema.json" }, "data_tests": { "$ref": "model.data_tests.schema.json" }, "materialization": { "$ref": "model.materialization.schema.json" }, "materialized": { "$ref": "model.materialized.schema.json" }, diff --git a/schemas/model.type.int_select_model.schema.json b/schemas/model.type.int_select_model.schema.json index 9902483..e1fdbc2 100644 --- a/schemas/model.type.int_select_model.schema.json +++ b/schemas/model.type.int_select_model.schema.json @@ -14,6 +14,7 @@ "lightdash": { "$ref": "model.lightdash.schema.json" }, "tags": { "$ref": "model.tags.schema.json" }, "meta": { "$ref": "model.meta.schema.json" }, + "depends_on": { "$ref": "model.depends_on.schema.json" }, "data_tests": { "$ref": "model.data_tests.schema.json" }, "materialization": { "$ref": "model.materialization.schema.json" }, "materialized": { "$ref": "model.materialized.schema.json" }, diff --git a/schemas/model.type.int_union_models.schema.json b/schemas/model.type.int_union_models.schema.json index 4bc50a9..8a92e5b 100644 --- a/schemas/model.type.int_union_models.schema.json +++ b/schemas/model.type.int_union_models.schema.json @@ -14,6 +14,7 @@ "lightdash": { "$ref": "model.lightdash.schema.json" }, "tags": { "$ref": "model.tags.schema.json" }, "meta": { "$ref": "model.meta.schema.json" }, + "depends_on": { "$ref": "model.depends_on.schema.json" }, "data_tests": { "$ref": "model.data_tests.schema.json" }, "materialization": { "$ref": "model.materialization.schema.json" }, "materialized": { "$ref": "model.materialized.schema.json" }, diff --git a/schemas/model.type.mart_join_models.schema.json b/schemas/model.type.mart_join_models.schema.json index 0a6a9c6..d9c00c8 100644 --- a/schemas/model.type.mart_join_models.schema.json +++ b/schemas/model.type.mart_join_models.schema.json @@ -14,6 +14,7 @@ "lightdash": { "$ref": "model.lightdash.schema.json" }, "tags": { "$ref": "model.tags.schema.json" }, "meta": { "$ref": "model.meta.schema.json" }, + "depends_on": { "$ref": "model.depends_on.schema.json" }, "group_by": { "$ref": "model.group_by.schema.json" }, "having": { "$ref": "model.having.schema.json" }, "exclude_portal_partition_columns": { diff --git a/schemas/model.type.mart_select_model.schema.json b/schemas/model.type.mart_select_model.schema.json index 21a1813..e3139ee 100644 --- a/schemas/model.type.mart_select_model.schema.json +++ b/schemas/model.type.mart_select_model.schema.json @@ -14,6 +14,7 @@ "lightdash": { "$ref": "model.lightdash.schema.json" }, "tags": { "$ref": "model.tags.schema.json" }, "meta": { "$ref": "model.meta.schema.json" }, + "depends_on": { "$ref": "model.depends_on.schema.json" }, "group_by": { "$ref": "model.group_by.schema.json" }, "exclude_portal_partition_columns": { "$ref": "model.exclude_portal_partition_columns.schema.json" diff --git a/schemas/model.type.stg_select_model.schema.json b/schemas/model.type.stg_select_model.schema.json index a454aad..ba2d9cd 100644 --- a/schemas/model.type.stg_select_model.schema.json +++ b/schemas/model.type.stg_select_model.schema.json @@ -13,6 +13,7 @@ "description": { "$ref": "model.description.schema.json" }, "tags": { "$ref": "model.tags.schema.json" }, "meta": { "$ref": "model.meta.schema.json" }, + "depends_on": { "$ref": "model.depends_on.schema.json" }, "materialization": { "$ref": "model.materialization.schema.json" }, "materialized": { "$ref": "model.materialized.schema.json" }, "incremental_strategy": { diff --git a/schemas/model.type.stg_select_source.schema.json b/schemas/model.type.stg_select_source.schema.json index c64e479..4c2c4d5 100644 --- a/schemas/model.type.stg_select_source.schema.json +++ b/schemas/model.type.stg_select_source.schema.json @@ -13,6 +13,7 @@ "description": { "$ref": "model.description.schema.json" }, "tags": { "$ref": "model.tags.schema.json" }, "meta": { "$ref": "model.meta.schema.json" }, + "depends_on": { "$ref": "model.depends_on.schema.json" }, "materialization": { "$ref": "model.materialization.schema.json" }, "materialized": { "$ref": "model.materialized.schema.json" }, "incremental_strategy": { diff --git a/schemas/model.type.stg_union_sources.schema.json b/schemas/model.type.stg_union_sources.schema.json index 0f3b3e2..85badfd 100644 --- a/schemas/model.type.stg_union_sources.schema.json +++ b/schemas/model.type.stg_union_sources.schema.json @@ -13,6 +13,7 @@ "description": { "$ref": "model.description.schema.json" }, "tags": { "$ref": "model.tags.schema.json" }, "meta": { "$ref": "model.meta.schema.json" }, + "depends_on": { "$ref": "model.depends_on.schema.json" }, "materialization": { "$ref": "model.materialization.schema.json" }, "materialized": { "$ref": "model.materialized.schema.json" }, "incremental_strategy": { diff --git a/src/services/framework/__tests__/depends-on.test.ts b/src/services/framework/__tests__/depends-on.test.ts new file mode 100644 index 0000000..11f9120 --- /dev/null +++ b/src/services/framework/__tests__/depends-on.test.ts @@ -0,0 +1,104 @@ +import { describe, expect, test } from '@jest/globals'; +import { frameworkGenerateModelOutput } from '@services/framework/utils'; +import { extractFrameworkDependencies } from '@services/sync/dependencyGraph'; +import type { FrameworkModel } from '@shared/framework/types'; + +import { createTestDJ, createTestProject } from './helpers'; + +const project = createTestProject({ + nodes: { + ['model.project.model_a']: { + columns: { + col_a: { + name: 'col_a', + data_type: 'varchar', + meta: { type: 'dim' }, + }, + }, + }, + ['model.project.processed_orders']: { + columns: { + order_id: { + name: 'order_id', + data_type: 'varchar', + meta: { type: 'dim' }, + }, + }, + }, + ['model.project.another_model']: { + columns: { + id: { + name: 'id', + data_type: 'varchar', + meta: { type: 'dim' }, + }, + }, + }, + }, +}); + +describe('depends_on SQL emission', () => { + test('emits one --depends_on comment per value before config', () => { + const modelJson = { + type: 'int_select_model', + group: 'ml', + topic: 'test', + name: 'with_forced_deps', + select: ['col_a'], + from: { model: 'model_a' }, + depends_on: ['processed_orders', 'another_model'], + } as FrameworkModel; + + const { sql } = frameworkGenerateModelOutput({ + dj: createTestDJ(), + modelJson, + project, + }); + + expect(sql).toContain( + `--depends_on: {{ ref('processed_orders') }}\n--depends_on: {{ ref('another_model') }}\n\n{{`, + ); + expect( + sql.indexOf("--depends_on: {{ ref('processed_orders') }}"), + ).toBeLessThan(sql.indexOf('config(')); + }); + + test('omits --depends_on lines when the field is absent', () => { + const modelJson = { + type: 'int_select_model', + group: 'ml', + topic: 'test', + name: 'without_forced_deps', + select: ['col_a'], + from: { model: 'model_a' }, + } as FrameworkModel; + + const { sql } = frameworkGenerateModelOutput({ + dj: createTestDJ(), + modelJson, + project, + }); + + expect(sql).not.toContain('--depends_on:'); + }); +}); + +describe('extractFrameworkDependencies with depends_on', () => { + test('includes authored depends_on names alongside from.model', () => { + const modelJson = { + type: 'int_select_model', + group: 'ml', + topic: 'test', + name: 'with_forced_deps', + select: ['col_a'], + from: { model: 'model_a' }, + depends_on: ['processed_orders', 'another_model'], + } as FrameworkModel; + + const deps = extractFrameworkDependencies(modelJson); + expect(deps).toEqual( + expect.arrayContaining(['model_a', 'processed_orders', 'another_model']), + ); + expect(deps).toHaveLength(3); + }); +}); diff --git a/src/services/framework/utils/sql-utils.ts b/src/services/framework/utils/sql-utils.ts index 1febc55..d520f23 100644 --- a/src/services/framework/utils/sql-utils.ts +++ b/src/services/framework/utils/sql-utils.ts @@ -2127,6 +2127,15 @@ export function frameworkGenerateModelOutput({ let sql = ''; + // Force extra dbt DAG edges for refs hidden from parse (e.g. inside {% if execute %}). + // Emit as `--depends_on:` (no space) so dbt's parser matches the documented form. + if ('depends_on' in modelJson && modelJson.depends_on?.length) { + for (const modelName of modelJson.depends_on) { + sql += `--depends_on: {{ ref('${modelName}') }}\n`; + } + sql += '\n'; + } + // Append comments const modelComments = [ ...modelFrom.comments, diff --git a/src/services/sync/dependencyGraph.ts b/src/services/sync/dependencyGraph.ts index 901b658..1717edb 100644 --- a/src/services/sync/dependencyGraph.ts +++ b/src/services/sync/dependencyGraph.ts @@ -125,6 +125,13 @@ export function extractFrameworkDependencies( } } + // Authored forced refs (--depends_on comments) also affect sync order. + if ('depends_on' in modelJson && modelJson.depends_on) { + for (const modelName of modelJson.depends_on) { + dependencies.add(modelName); + } + } + return Array.from(dependencies); } diff --git a/src/shared/schema/types/model.depends_on.schema.d.ts b/src/shared/schema/types/model.depends_on.schema.d.ts new file mode 100644 index 0000000..ee01ec8 --- /dev/null +++ b/src/shared/schema/types/model.depends_on.schema.d.ts @@ -0,0 +1,17 @@ +/* eslint-disable */ +/** + * This file was automatically generated by json-schema-to-typescript. + * DO NOT MODIFY IT BY HAND. Instead, modify the source JSONSchema file, + * and run json-schema-to-typescript to regenerate this file. + */ + +/** + * Additional model names whose ref() edges should be forced in generated SQL via --depends_on comments (for refs hidden from dbt parse, e.g. inside {% if execute %}) + * + * @minItems 1 + */ +export type SchemaModelDependsOn = [SchemaModelRef, ...SchemaModelRef[]]; +/** + * Validate model ids + */ +export type SchemaModelRef = string; diff --git a/src/shared/schema/types/model.schema.d.ts b/src/shared/schema/types/model.schema.d.ts index 95a1510..af5b435 100644 --- a/src/shared/schema/types/model.schema.d.ts +++ b/src/shared/schema/types/model.schema.d.ts @@ -46,6 +46,16 @@ export type SchemaModelTags = ( type?: 'ai_hints' | 'exclude' | 'inherit' | 'local'; } )[]; +/** + * Additional model names whose ref() edges should be forced in generated SQL via --depends_on comments (for refs hidden from dbt parse, e.g. inside {% if execute %}) + * + * @minItems 1 + */ +export type SchemaModelDependsOn = [SchemaModelRef, ...SchemaModelRef[]]; +/** + * Validate model ids + */ +export type SchemaModelRef = string; /** * Materialization Configuration. The incremental object form accepts `format` (delta_lake | hive | iceberg), `partitions`, `strategy`, `database`, and the physical-layout tuning fields `bucket` and `sorted_by`. DJ translates `bucket` / `sorted_by` into the correct Trino table properties per format: Iceberg emits bucket transforms inside `partitioning` plus a standalone `sorted_by`; Hive / Glue emits `bucketed_by` + `bucket_count` + `sorted_by`; Delta Lake supports neither and DJ flags it. */ @@ -358,10 +368,6 @@ export type SchemaModelSelectExpr = data_tests?: SchemaColumnDataTests; type: 'fct'; }; -/** - * Validate model ids - */ -export type SchemaModelRef = string; /** * Schema when selecting columns from a source */ @@ -904,6 +910,7 @@ export interface SchemaModelTypeStgSelectSource { description?: SchemaModelDescription; tags?: SchemaModelTags; meta?: SchemaModelMeta; + depends_on?: SchemaModelDependsOn; materialization?: SchemaModelMaterialization; materialized?: SchemaModelMaterialized; incremental_strategy?: SchemaModelIncrementalStrategyDeprecatedTopLevelField; @@ -1231,6 +1238,7 @@ export interface SchemaModelTypeStgSelectModel { description?: SchemaModelDescription; tags?: SchemaModelTags; meta?: SchemaModelMeta; + depends_on?: SchemaModelDependsOn; materialization?: SchemaModelMaterialization; materialized?: SchemaModelMaterialized; incremental_strategy?: SchemaModelIncrementalStrategyDeprecatedTopLevelField; @@ -1277,6 +1285,7 @@ export interface SchemaModelTypeStgUnionSources { description?: SchemaModelDescription; tags?: SchemaModelTags; meta?: SchemaModelMeta; + depends_on?: SchemaModelDependsOn; materialization?: SchemaModelMaterialization; materialized?: SchemaModelMaterialized; incremental_strategy?: SchemaModelIncrementalStrategyDeprecatedTopLevelField; @@ -1329,6 +1338,7 @@ export interface SchemaModelTypeIntSelectModel { lightdash?: SchemaModelLightdash; tags?: SchemaModelTags; meta?: SchemaModelMeta; + depends_on?: SchemaModelDependsOn; data_tests?: SchemaModelDataTests; materialization?: SchemaModelMaterialization; materialized?: SchemaModelMaterialized; @@ -1538,6 +1548,7 @@ export interface SchemaModelTypeIntJoinColumn { lightdash?: SchemaModelLightdash; tags?: SchemaModelTags; meta?: SchemaModelMeta; + depends_on?: SchemaModelDependsOn; data_tests?: SchemaModelDataTests; materialization?: SchemaModelMaterialization; materialized?: SchemaModelMaterialized; @@ -1611,6 +1622,7 @@ export interface SchemaModelTypeIntJoinModels { lightdash?: SchemaModelLightdash; tags?: SchemaModelTags; meta?: SchemaModelMeta; + depends_on?: SchemaModelDependsOn; data_tests?: SchemaModelDataTests; materialization?: SchemaModelMaterialization; materialized?: SchemaModelMaterialized; @@ -1679,6 +1691,7 @@ export interface SchemaModelTypeIntLookbackModel { lightdash?: SchemaModelLightdash; tags?: SchemaModelTags; meta?: SchemaModelMeta; + depends_on?: SchemaModelDependsOn; data_tests?: SchemaModelDataTests; materialization?: SchemaModelMaterialization; materialized?: SchemaModelMaterialized; @@ -1741,6 +1754,7 @@ export interface SchemaModelTypeIntRollupModel { lightdash?: SchemaModelLightdash; tags?: SchemaModelTags; meta?: SchemaModelMeta; + depends_on?: SchemaModelDependsOn; data_tests?: SchemaModelDataTests; materialization?: SchemaModelMaterialization; materialized?: SchemaModelMaterialized; @@ -1770,6 +1784,7 @@ export interface SchemaModelTypeIntUnionModels { lightdash?: SchemaModelLightdash; tags?: SchemaModelTags; meta?: SchemaModelMeta; + depends_on?: SchemaModelDependsOn; data_tests?: SchemaModelDataTests; materialization?: SchemaModelMaterialization; materialized?: SchemaModelMaterialized; @@ -1831,6 +1846,7 @@ export interface SchemaModelTypeMartSelectModel { lightdash?: SchemaModelLightdash; tags?: SchemaModelTags; meta?: SchemaModelMeta; + depends_on?: SchemaModelDependsOn; group_by?: SchemaModelGroupBy; exclude_portal_partition_columns?: SchemaModelExcludePortalPartitionColumns; exclude_datetime?: SchemaModelExcludeDatetime; @@ -1881,6 +1897,7 @@ export interface SchemaModelTypeMartJoinModels { lightdash?: SchemaModelLightdash; tags?: SchemaModelTags; meta?: SchemaModelMeta; + depends_on?: SchemaModelDependsOn; group_by?: SchemaModelGroupBy; having?: SchemaModelHaving; exclude_portal_partition_columns?: SchemaModelExcludePortalPartitionColumns; diff --git a/src/shared/schema/types/model.type.int_join_column.schema.d.ts b/src/shared/schema/types/model.type.int_join_column.schema.d.ts index 001bf25..cf560e4 100644 --- a/src/shared/schema/types/model.type.int_join_column.schema.d.ts +++ b/src/shared/schema/types/model.type.int_join_column.schema.d.ts @@ -53,6 +53,16 @@ export type SchemaModelTags = ( type?: 'ai_hints' | 'exclude' | 'inherit' | 'local'; } )[]; +/** + * Additional model names whose ref() edges should be forced in generated SQL via --depends_on comments (for refs hidden from dbt parse, e.g. inside {% if execute %}) + * + * @minItems 1 + */ +export type SchemaModelDependsOn = [SchemaModelRef, ...SchemaModelRef[]]; +/** + * Validate model ids + */ +export type SchemaModelRef = string; /** * Validate model data_tests */ @@ -315,10 +325,6 @@ export type SchemaModelWhere = * SQL expression to be used when selecting the column (name will be the alias */ export type SchemaColumnExpr = string; -/** - * Validate model ids - */ -export type SchemaModelRef = string; /** * Validate source ids */ @@ -359,6 +365,7 @@ export interface SchemaModelTypeIntJoinColumn { lightdash?: SchemaModelLightdash; tags?: SchemaModelTags; meta?: SchemaModelMeta; + depends_on?: SchemaModelDependsOn; data_tests?: SchemaModelDataTests; materialization?: SchemaModelMaterialization; materialized?: SchemaModelMaterialized; diff --git a/src/shared/schema/types/model.type.int_join_models.schema.d.ts b/src/shared/schema/types/model.type.int_join_models.schema.d.ts index a4e670c..52eaec2 100644 --- a/src/shared/schema/types/model.type.int_join_models.schema.d.ts +++ b/src/shared/schema/types/model.type.int_join_models.schema.d.ts @@ -53,6 +53,16 @@ export type SchemaModelTags = ( type?: 'ai_hints' | 'exclude' | 'inherit' | 'local'; } )[]; +/** + * Additional model names whose ref() edges should be forced in generated SQL via --depends_on comments (for refs hidden from dbt parse, e.g. inside {% if execute %}) + * + * @minItems 1 + */ +export type SchemaModelDependsOn = [SchemaModelRef, ...SchemaModelRef[]]; +/** + * Validate model ids + */ +export type SchemaModelRef = string; /** * Validate model data_tests */ @@ -317,10 +327,6 @@ export type SchemaModelHaving = subquery?: SchemaModelSubquery; }[]; }; -/** - * Validate model ids - */ -export type SchemaModelRef = string; /** * Validate source ids */ @@ -860,6 +866,7 @@ export interface SchemaModelTypeIntJoinModels { lightdash?: SchemaModelLightdash; tags?: SchemaModelTags; meta?: SchemaModelMeta; + depends_on?: SchemaModelDependsOn; data_tests?: SchemaModelDataTests; materialization?: SchemaModelMaterialization; materialized?: SchemaModelMaterialized; diff --git a/src/shared/schema/types/model.type.int_lookback_model.schema.d.ts b/src/shared/schema/types/model.type.int_lookback_model.schema.d.ts index 14b293f..8795681 100644 --- a/src/shared/schema/types/model.type.int_lookback_model.schema.d.ts +++ b/src/shared/schema/types/model.type.int_lookback_model.schema.d.ts @@ -53,6 +53,16 @@ export type SchemaModelTags = ( type?: 'ai_hints' | 'exclude' | 'inherit' | 'local'; } )[]; +/** + * Additional model names whose ref() edges should be forced in generated SQL via --depends_on comments (for refs hidden from dbt parse, e.g. inside {% if execute %}) + * + * @minItems 1 + */ +export type SchemaModelDependsOn = [SchemaModelRef, ...SchemaModelRef[]]; +/** + * Validate model ids + */ +export type SchemaModelRef = string; /** * Validate model data_tests */ @@ -497,10 +507,6 @@ export type SchemaModelSelectModel = model: SchemaModelRef; override_prefix?: SchemaColumnName; }; -/** - * Validate model ids - */ -export type SchemaModelRef = string; /** * Schema when selecting columns from another model with an agg applied */ @@ -556,6 +562,7 @@ export interface SchemaModelTypeIntLookbackModel { lightdash?: SchemaModelLightdash; tags?: SchemaModelTags; meta?: SchemaModelMeta; + depends_on?: SchemaModelDependsOn; data_tests?: SchemaModelDataTests; materialization?: SchemaModelMaterialization; materialized?: SchemaModelMaterialized; diff --git a/src/shared/schema/types/model.type.int_rollup_model.schema.d.ts b/src/shared/schema/types/model.type.int_rollup_model.schema.d.ts index 1705eee..14a7039 100644 --- a/src/shared/schema/types/model.type.int_rollup_model.schema.d.ts +++ b/src/shared/schema/types/model.type.int_rollup_model.schema.d.ts @@ -53,6 +53,16 @@ export type SchemaModelTags = ( type?: 'ai_hints' | 'exclude' | 'inherit' | 'local'; } )[]; +/** + * Additional model names whose ref() edges should be forced in generated SQL via --depends_on comments (for refs hidden from dbt parse, e.g. inside {% if execute %}) + * + * @minItems 1 + */ +export type SchemaModelDependsOn = [SchemaModelRef, ...SchemaModelRef[]]; +/** + * Validate model ids + */ +export type SchemaModelRef = string; /** * Validate model data_tests */ @@ -263,10 +273,6 @@ export type SchemaModelExcludeFrameworkArtifacts = 'all' | 'columns'; * Will prevent the automatic portal source count column from getting added */ export type SchemaModelExcludePortalSourceCount = boolean; -/** - * Validate model ids - */ -export type SchemaModelRef = string; /** * Validates schema for rollup models @@ -280,6 +286,7 @@ export interface SchemaModelTypeIntRollupModel { lightdash?: SchemaModelLightdash; tags?: SchemaModelTags; meta?: SchemaModelMeta; + depends_on?: SchemaModelDependsOn; data_tests?: SchemaModelDataTests; materialization?: SchemaModelMaterialization; materialized?: SchemaModelMaterialized; diff --git a/src/shared/schema/types/model.type.int_select_model.schema.d.ts b/src/shared/schema/types/model.type.int_select_model.schema.d.ts index 8871fc3..9267015 100644 --- a/src/shared/schema/types/model.type.int_select_model.schema.d.ts +++ b/src/shared/schema/types/model.type.int_select_model.schema.d.ts @@ -53,6 +53,16 @@ export type SchemaModelTags = ( type?: 'ai_hints' | 'exclude' | 'inherit' | 'local'; } )[]; +/** + * Additional model names whose ref() edges should be forced in generated SQL via --depends_on comments (for refs hidden from dbt parse, e.g. inside {% if execute %}) + * + * @minItems 1 + */ +export type SchemaModelDependsOn = [SchemaModelRef, ...SchemaModelRef[]]; +/** + * Validate model ids + */ +export type SchemaModelRef = string; /** * Validate model data_tests */ @@ -317,10 +327,6 @@ export type SchemaModelHaving = subquery?: SchemaModelSubquery; }[]; }; -/** - * Validate model ids - */ -export type SchemaModelRef = string; /** * Validate source ids */ @@ -860,6 +866,7 @@ export interface SchemaModelTypeIntSelectModel { lightdash?: SchemaModelLightdash; tags?: SchemaModelTags; meta?: SchemaModelMeta; + depends_on?: SchemaModelDependsOn; data_tests?: SchemaModelDataTests; materialization?: SchemaModelMaterialization; materialized?: SchemaModelMaterialized; diff --git a/src/shared/schema/types/model.type.int_union_models.schema.d.ts b/src/shared/schema/types/model.type.int_union_models.schema.d.ts index b259d7e..b3c29cc 100644 --- a/src/shared/schema/types/model.type.int_union_models.schema.d.ts +++ b/src/shared/schema/types/model.type.int_union_models.schema.d.ts @@ -53,6 +53,16 @@ export type SchemaModelTags = ( type?: 'ai_hints' | 'exclude' | 'inherit' | 'local'; } )[]; +/** + * Additional model names whose ref() edges should be forced in generated SQL via --depends_on comments (for refs hidden from dbt parse, e.g. inside {% if execute %}) + * + * @minItems 1 + */ +export type SchemaModelDependsOn = [SchemaModelRef, ...SchemaModelRef[]]; +/** + * Validate model ids + */ +export type SchemaModelRef = string; /** * Validate model data_tests */ @@ -269,10 +279,6 @@ export type SchemaModelExcludePortalSourceCount = boolean; * @minItems 1 */ export type SchemaModelCTEs = [SchemaModelCTE, ...SchemaModelCTE[]]; -/** - * Validate model ids - */ -export type SchemaModelRef = string; /** * Validates the join argument when joining multiple models or CTEs * @@ -806,6 +812,7 @@ export interface SchemaModelTypeIntUnionModels { lightdash?: SchemaModelLightdash; tags?: SchemaModelTags; meta?: SchemaModelMeta; + depends_on?: SchemaModelDependsOn; data_tests?: SchemaModelDataTests; materialization?: SchemaModelMaterialization; materialized?: SchemaModelMaterialized; diff --git a/src/shared/schema/types/model.type.mart_join_models.schema.d.ts b/src/shared/schema/types/model.type.mart_join_models.schema.d.ts index fbca3d2..2f72672 100644 --- a/src/shared/schema/types/model.type.mart_join_models.schema.d.ts +++ b/src/shared/schema/types/model.type.mart_join_models.schema.d.ts @@ -53,6 +53,16 @@ export type SchemaModelTags = ( type?: 'ai_hints' | 'exclude' | 'inherit' | 'local'; } )[]; +/** + * Additional model names whose ref() edges should be forced in generated SQL via --depends_on comments (for refs hidden from dbt parse, e.g. inside {% if execute %}) + * + * @minItems 1 + */ +export type SchemaModelDependsOn = [SchemaModelRef, ...SchemaModelRef[]]; +/** + * Validate model ids + */ +export type SchemaModelRef = string; /** * Validate model group by */ @@ -107,10 +117,6 @@ export type SchemaModelHaving = subquery?: SchemaModelSubquery; }[]; }; -/** - * Validate model ids - */ -export type SchemaModelRef = string; /** * Validate source ids */ @@ -662,6 +668,7 @@ export interface SchemaModelTypeMartJoinModels { lightdash?: SchemaModelLightdash; tags?: SchemaModelTags; meta?: SchemaModelMeta; + depends_on?: SchemaModelDependsOn; group_by?: SchemaModelGroupBy; having?: SchemaModelHaving; exclude_portal_partition_columns?: SchemaModelExcludePortalPartitionColumns; diff --git a/src/shared/schema/types/model.type.mart_select_model.schema.d.ts b/src/shared/schema/types/model.type.mart_select_model.schema.d.ts index 60c6233..5c9c7cb 100644 --- a/src/shared/schema/types/model.type.mart_select_model.schema.d.ts +++ b/src/shared/schema/types/model.type.mart_select_model.schema.d.ts @@ -53,6 +53,16 @@ export type SchemaModelTags = ( type?: 'ai_hints' | 'exclude' | 'inherit' | 'local'; } )[]; +/** + * Additional model names whose ref() edges should be forced in generated SQL via --depends_on comments (for refs hidden from dbt parse, e.g. inside {% if execute %}) + * + * @minItems 1 + */ +export type SchemaModelDependsOn = [SchemaModelRef, ...SchemaModelRef[]]; +/** + * Validate model ids + */ +export type SchemaModelRef = string; /** * Validate model group by */ @@ -113,10 +123,6 @@ export type SchemaModelExcludePortalSourceCount = boolean; * @minItems 1 */ export type SchemaModelCTEs = [SchemaModelCTE, ...SchemaModelCTE[]]; -/** - * Validate model ids - */ -export type SchemaModelRef = string; /** * Validates the join argument when joining multiple models or CTEs * @@ -662,6 +668,7 @@ export interface SchemaModelTypeMartSelectModel { lightdash?: SchemaModelLightdash; tags?: SchemaModelTags; meta?: SchemaModelMeta; + depends_on?: SchemaModelDependsOn; group_by?: SchemaModelGroupBy; exclude_portal_partition_columns?: SchemaModelExcludePortalPartitionColumns; exclude_datetime?: SchemaModelExcludeDatetime; diff --git a/src/shared/schema/types/model.type.stg_select_model.schema.d.ts b/src/shared/schema/types/model.type.stg_select_model.schema.d.ts index 40fc66f..52e07d0 100644 --- a/src/shared/schema/types/model.type.stg_select_model.schema.d.ts +++ b/src/shared/schema/types/model.type.stg_select_model.schema.d.ts @@ -31,6 +31,16 @@ export type SchemaModelTags = ( type?: 'ai_hints' | 'exclude' | 'inherit' | 'local'; } )[]; +/** + * Additional model names whose ref() edges should be forced in generated SQL via --depends_on comments (for refs hidden from dbt parse, e.g. inside {% if execute %}) + * + * @minItems 1 + */ +export type SchemaModelDependsOn = [SchemaModelRef, ...SchemaModelRef[]]; +/** + * Validate model ids + */ +export type SchemaModelRef = string; /** * Materialization Configuration. The incremental object form accepts `format` (delta_lake | hive | iceberg), `partitions`, `strategy`, `database`, and the physical-layout tuning fields `bucket` and `sorted_by`. DJ translates `bucket` / `sorted_by` into the correct Trino table properties per format: Iceberg emits bucket transforms inside `partitioning` plus a standalone `sorted_by`; Hive / Glue emits `bucketed_by` + `bucket_count` + `sorted_by`; Delta Lake supports neither and DJ flags it. */ @@ -207,10 +217,6 @@ export type SchemaModelWhere = * SQL expression to be used when selecting the column (name will be the alias */ export type SchemaColumnExpr = string; -/** - * Validate model ids - */ -export type SchemaModelRef = string; /** * Validate source ids */ @@ -422,6 +428,7 @@ export interface SchemaModelTypeStgSelectModel { description?: SchemaModelDescription; tags?: SchemaModelTags; meta?: SchemaModelMeta; + depends_on?: SchemaModelDependsOn; materialization?: SchemaModelMaterialization; materialized?: SchemaModelMaterialized; incremental_strategy?: SchemaModelIncrementalStrategyDeprecatedTopLevelField; diff --git a/src/shared/schema/types/model.type.stg_select_source.schema.d.ts b/src/shared/schema/types/model.type.stg_select_source.schema.d.ts index 88157c0..98b435b 100644 --- a/src/shared/schema/types/model.type.stg_select_source.schema.d.ts +++ b/src/shared/schema/types/model.type.stg_select_source.schema.d.ts @@ -31,6 +31,16 @@ export type SchemaModelTags = ( type?: 'ai_hints' | 'exclude' | 'inherit' | 'local'; } )[]; +/** + * Additional model names whose ref() edges should be forced in generated SQL via --depends_on comments (for refs hidden from dbt parse, e.g. inside {% if execute %}) + * + * @minItems 1 + */ +export type SchemaModelDependsOn = [SchemaModelRef, ...SchemaModelRef[]]; +/** + * Validate model ids + */ +export type SchemaModelRef = string; /** * Materialization Configuration. The incremental object form accepts `format` (delta_lake | hive | iceberg), `partitions`, `strategy`, `database`, and the physical-layout tuning fields `bucket` and `sorted_by`. DJ translates `bucket` / `sorted_by` into the correct Trino table properties per format: Iceberg emits bucket transforms inside `partitioning` plus a standalone `sorted_by`; Hive / Glue emits `bucketed_by` + `bucket_count` + `sorted_by`; Delta Lake supports neither and DJ flags it. */ @@ -343,10 +353,6 @@ export type SchemaModelSelectExpr = data_tests?: SchemaColumnDataTests; type: 'fct'; }; -/** - * Validate model ids - */ -export type SchemaModelRef = string; /** * Schema when selecting columns from a source */ @@ -412,6 +418,7 @@ export interface SchemaModelTypeStgSelectSource { description?: SchemaModelDescription; tags?: SchemaModelTags; meta?: SchemaModelMeta; + depends_on?: SchemaModelDependsOn; materialization?: SchemaModelMaterialization; materialized?: SchemaModelMaterialized; incremental_strategy?: SchemaModelIncrementalStrategyDeprecatedTopLevelField; diff --git a/src/shared/schema/types/model.type.stg_union_sources.schema.d.ts b/src/shared/schema/types/model.type.stg_union_sources.schema.d.ts index 5be4f5c..68f5a04 100644 --- a/src/shared/schema/types/model.type.stg_union_sources.schema.d.ts +++ b/src/shared/schema/types/model.type.stg_union_sources.schema.d.ts @@ -31,6 +31,16 @@ export type SchemaModelTags = ( type?: 'ai_hints' | 'exclude' | 'inherit' | 'local'; } )[]; +/** + * Additional model names whose ref() edges should be forced in generated SQL via --depends_on comments (for refs hidden from dbt parse, e.g. inside {% if execute %}) + * + * @minItems 1 + */ +export type SchemaModelDependsOn = [SchemaModelRef, ...SchemaModelRef[]]; +/** + * Validate model ids + */ +export type SchemaModelRef = string; /** * Materialization Configuration. The incremental object form accepts `format` (delta_lake | hive | iceberg), `partitions`, `strategy`, `database`, and the physical-layout tuning fields `bucket` and `sorted_by`. DJ translates `bucket` / `sorted_by` into the correct Trino table properties per format: Iceberg emits bucket transforms inside `partitioning` plus a standalone `sorted_by`; Hive / Glue emits `bucketed_by` + `bucket_count` + `sorted_by`; Delta Lake supports neither and DJ flags it. */ @@ -343,10 +353,6 @@ export type SchemaModelSelectExpr = data_tests?: SchemaColumnDataTests; type: 'fct'; }; -/** - * Validate model ids - */ -export type SchemaModelRef = string; /** * Schema when selecting columns from a source */ @@ -412,6 +418,7 @@ export interface SchemaModelTypeStgUnionSources { description?: SchemaModelDescription; tags?: SchemaModelTags; meta?: SchemaModelMeta; + depends_on?: SchemaModelDependsOn; materialization?: SchemaModelMaterialization; materialized?: SchemaModelMaterialized; incremental_strategy?: SchemaModelIncrementalStrategyDeprecatedTopLevelField; diff --git a/templates/_agents-dj/reference/model-types.md b/templates/_agents-dj/reference/model-types.md index f52d979..011ea8a 100644 --- a/templates/_agents-dj/reference/model-types.md +++ b/templates/_agents-dj/reference/model-types.md @@ -421,6 +421,7 @@ Named column from a CTE: | ---------------------------------- | ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `description` | string | Model description | | `tags` | array | Tags for categorization, e.g. `["my_tag", "my_group"]` | +| `depends_on` | array | Extra model names to force as dbt DAG edges via `--depends_on: {{ ref('...') }}` SQL comments (for refs hidden from parse, e.g. inside `{% if execute %}`) | | `materialized` | string | Legacy: `"incremental"` or `"ephemeral"` (default is view-like). Prefer `materialization` instead. | | `materialization` | string/object | Preferred. String `"incremental"` or `"ephemeral"`, or object `{ "type": "incremental", "format"?, "partitions"?, "strategy"?, "database"? }`. See [materialization.md](materialization.md). | | `incremental_strategy` | object | Legacy: `{ "type": "delete+insert" }` or `{ "type": "merge", "unique_key": "id" }`. Prefer `materialization.strategy`. |