v2.3.0

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Change Log
 
+## 2.3.0
+
+### Framework
+
+- **Force extra dbt model dependencies with `depends_on`.** On any `.model.json`, set `"depends_on": ["processed_orders", ...]` to emit `--depends_on: {{ ref('...') }}` comments in generated SQL so dbt can discover refs that are otherwise hidden at parse time (for example inside `{% if execute %}`).
+
 ## 2.2.1
 
 ### Security

docs/models/README.md

@@ -417,6 +417,23 @@ Understanding the relationship between model types:
 - **Intermediate models** feed **mart models**
 - **Mart models** feed **BI tools and dashboards**
 
+### Forcing dbt dependencies
+
+dbt builds its DAG from `ref()` calls it can see at parse time. When a dependency is only used inside a branch that is skipped during parse (for example `{% if execute %}`), add the upstream model names under `depends_on` on the `.model.json`. DJ emits one SQL comment per value:
+
+```jsonc
+{
+  "depends_on": ["processed_orders", "another_model"]
+}
+```
+
+```sql
+--depends_on: {{ ref('processed_orders') }}
+--depends_on: {{ ref('another_model') }}
+```
+
+See [dbt: Forcing dependencies](https://docs.getdbt.com/reference/dbt-jinja-functions/ref?version=2#forcing-dependencies).
+
 ### Performance Optimization
 
 - **Materialization**: Choose appropriate strategy for data volume

package.json

@@ -10,7 +10,7 @@
     "type": "git",
     "url": "https://github.com/Workday/dj.git"
   },
-  "version": "2.2.1",
+  "version": "2.3.0",
   "workspaces": [
     "web"
   ],

schemas/model.depends_on.schema.json

@@ -0,0 +1,11 @@
+{
+  "$id": "model.depends_on.schema.json",
+  "title": "Schema Model Depends On",
+  "description": "Additional model names whose ref() edges should be forced in generated SQL via --depends_on comments (for refs hidden from dbt parse, e.g. inside {% if execute %})",
+  "type": "array",
+  "minItems": 1,
+  "uniqueItems": true,
+  "items": {
+    "$ref": "model.ref.schema.json"
+  }
+}

schemas/model.type.int_join_column.schema.json

@@ -30,6 +30,9 @@
     "meta": {
       "$ref": "model.meta.schema.json"
     },
+    "depends_on": {
+      "$ref": "model.depends_on.schema.json"
+    },
     "data_tests": {
       "$ref": "model.data_tests.schema.json"
     },

schemas/model.type.int_join_models.schema.json

@@ -14,6 +14,7 @@
     "lightdash": { "$ref": "model.lightdash.schema.json" },
     "tags": { "$ref": "model.tags.schema.json" },
     "meta": { "$ref": "model.meta.schema.json" },
+    "depends_on": { "$ref": "model.depends_on.schema.json" },
     "data_tests": { "$ref": "model.data_tests.schema.json" },
     "materialization": { "$ref": "model.materialization.schema.json" },
     "materialized": { "$ref": "model.materialized.schema.json" },

schemas/model.type.int_lookback_model.schema.json

@@ -14,6 +14,7 @@
     "lightdash": { "$ref": "model.lightdash.schema.json" },
     "tags": { "$ref": "model.tags.schema.json" },
     "meta": { "$ref": "model.meta.schema.json" },
+    "depends_on": { "$ref": "model.depends_on.schema.json" },
     "data_tests": { "$ref": "model.data_tests.schema.json" },
     "materialization": { "$ref": "model.materialization.schema.json" },
     "materialized": { "$ref": "model.materialized.schema.json" },

schemas/model.type.int_rollup_model.schema.json

@@ -14,6 +14,7 @@
     "lightdash": { "$ref": "model.lightdash.schema.json" },
     "tags": { "$ref": "model.tags.schema.json" },
     "meta": { "$ref": "model.meta.schema.json" },
+    "depends_on": { "$ref": "model.depends_on.schema.json" },
     "data_tests": { "$ref": "model.data_tests.schema.json" },
     "materialization": { "$ref": "model.materialization.schema.json" },
     "materialized": { "$ref": "model.materialized.schema.json" },

schemas/model.type.int_select_model.schema.json

@@ -14,6 +14,7 @@
     "lightdash": { "$ref": "model.lightdash.schema.json" },
     "tags": { "$ref": "model.tags.schema.json" },
     "meta": { "$ref": "model.meta.schema.json" },
+    "depends_on": { "$ref": "model.depends_on.schema.json" },
     "data_tests": { "$ref": "model.data_tests.schema.json" },
     "materialization": { "$ref": "model.materialization.schema.json" },
     "materialized": { "$ref": "model.materialized.schema.json" },

schemas/model.type.int_union_models.schema.json

@@ -14,6 +14,7 @@
     "lightdash": { "$ref": "model.lightdash.schema.json" },
     "tags": { "$ref": "model.tags.schema.json" },
     "meta": { "$ref": "model.meta.schema.json" },
+    "depends_on": { "$ref": "model.depends_on.schema.json" },
     "data_tests": { "$ref": "model.data_tests.schema.json" },
     "materialization": { "$ref": "model.materialization.schema.json" },
     "materialized": { "$ref": "model.materialized.schema.json" },

schemas/model.type.mart_join_models.schema.json

@@ -14,6 +14,7 @@
     "lightdash": { "$ref": "model.lightdash.schema.json" },
     "tags": { "$ref": "model.tags.schema.json" },
     "meta": { "$ref": "model.meta.schema.json" },
+    "depends_on": { "$ref": "model.depends_on.schema.json" },
     "group_by": { "$ref": "model.group_by.schema.json" },
     "having": { "$ref": "model.having.schema.json" },
     "exclude_portal_partition_columns": {

schemas/model.type.mart_select_model.schema.json

@@ -14,6 +14,7 @@
     "lightdash": { "$ref": "model.lightdash.schema.json" },
     "tags": { "$ref": "model.tags.schema.json" },
     "meta": { "$ref": "model.meta.schema.json" },
+    "depends_on": { "$ref": "model.depends_on.schema.json" },
     "group_by": { "$ref": "model.group_by.schema.json" },
     "exclude_portal_partition_columns": {
       "$ref": "model.exclude_portal_partition_columns.schema.json"

schemas/model.type.stg_select_model.schema.json

@@ -13,6 +13,7 @@
     "description": { "$ref": "model.description.schema.json" },
     "tags": { "$ref": "model.tags.schema.json" },
     "meta": { "$ref": "model.meta.schema.json" },
+    "depends_on": { "$ref": "model.depends_on.schema.json" },
     "materialization": { "$ref": "model.materialization.schema.json" },
     "materialized": { "$ref": "model.materialized.schema.json" },
     "incremental_strategy": {

schemas/model.type.stg_select_source.schema.json

@@ -13,6 +13,7 @@
     "description": { "$ref": "model.description.schema.json" },
     "tags": { "$ref": "model.tags.schema.json" },
     "meta": { "$ref": "model.meta.schema.json" },
+    "depends_on": { "$ref": "model.depends_on.schema.json" },
     "materialization": { "$ref": "model.materialization.schema.json" },
     "materialized": { "$ref": "model.materialized.schema.json" },
     "incremental_strategy": {

schemas/model.type.stg_union_sources.schema.json

@@ -13,6 +13,7 @@
     "description": { "$ref": "model.description.schema.json" },
     "tags": { "$ref": "model.tags.schema.json" },
     "meta": { "$ref": "model.meta.schema.json" },
+    "depends_on": { "$ref": "model.depends_on.schema.json" },
     "materialization": { "$ref": "model.materialization.schema.json" },
     "materialized": { "$ref": "model.materialized.schema.json" },
     "incremental_strategy": {

src/services/framework/__tests__/depends-on.test.ts

@@ -0,0 +1,104 @@
+import { describe, expect, test } from '@jest/globals';
+import { frameworkGenerateModelOutput } from '@services/framework/utils';
+import { extractFrameworkDependencies } from '@services/sync/dependencyGraph';
+import type { FrameworkModel } from '@shared/framework/types';
+
+import { createTestDJ, createTestProject } from './helpers';
+
+const project = createTestProject({
+  nodes: {
+    ['model.project.model_a']: {
+      columns: {
+        col_a: {
+          name: 'col_a',
+          data_type: 'varchar',
+          meta: { type: 'dim' },
+        },
+      },
+    },
+    ['model.project.processed_orders']: {
+      columns: {
+        order_id: {
+          name: 'order_id',
+          data_type: 'varchar',
+          meta: { type: 'dim' },
+        },
+      },
+    },
+    ['model.project.another_model']: {
+      columns: {
+        id: {
+          name: 'id',
+          data_type: 'varchar',
+          meta: { type: 'dim' },
+        },
+      },
+    },
+  },
+});
+
+describe('depends_on SQL emission', () => {
+  test('emits one --depends_on comment per value before config', () => {
+    const modelJson = {
+      type: 'int_select_model',
+      group: 'ml',
+      topic: 'test',
+      name: 'with_forced_deps',
+      select: ['col_a'],
+      from: { model: 'model_a' },
+      depends_on: ['processed_orders', 'another_model'],
+    } as FrameworkModel;
+
+    const { sql } = frameworkGenerateModelOutput({
+      dj: createTestDJ(),
+      modelJson,
+      project,
+    });
+
+    expect(sql).toContain(
+      `--depends_on: {{ ref('processed_orders') }}\n--depends_on: {{ ref('another_model') }}\n\n{{`,
+    );
+    expect(
+      sql.indexOf("--depends_on: {{ ref('processed_orders') }}"),
+    ).toBeLessThan(sql.indexOf('config('));
+  });
+
+  test('omits --depends_on lines when the field is absent', () => {
+    const modelJson = {
+      type: 'int_select_model',
+      group: 'ml',
+      topic: 'test',
+      name: 'without_forced_deps',
+      select: ['col_a'],
+      from: { model: 'model_a' },
+    } as FrameworkModel;
+
+    const { sql } = frameworkGenerateModelOutput({
+      dj: createTestDJ(),
+      modelJson,
+      project,
+    });
+
+    expect(sql).not.toContain('--depends_on:');
+  });
+});
+
+describe('extractFrameworkDependencies with depends_on', () => {
+  test('includes authored depends_on names alongside from.model', () => {
+    const modelJson = {
+      type: 'int_select_model',
+      group: 'ml',
+      topic: 'test',
+      name: 'with_forced_deps',
+      select: ['col_a'],
+      from: { model: 'model_a' },
+      depends_on: ['processed_orders', 'another_model'],
+    } as FrameworkModel;
+
+    const deps = extractFrameworkDependencies(modelJson);
+    expect(deps).toEqual(
+      expect.arrayContaining(['model_a', 'processed_orders', 'another_model']),
+    );
+    expect(deps).toHaveLength(3);
+  });
+});

src/services/framework/utils/sql-utils.ts

@@ -2127,6 +2127,15 @@ export function frameworkGenerateModelOutput({
 
   let sql = '';
 
+  // Force extra dbt DAG edges for refs hidden from parse (e.g. inside {% if execute %}).
+  // Emit as `--depends_on:` (no space) so dbt's parser matches the documented form.
+  if ('depends_on' in modelJson && modelJson.depends_on?.length) {
+    for (const modelName of modelJson.depends_on) {
+      sql += `--depends_on: {{ ref('${modelName}') }}\n`;
+    }
+    sql += '\n';
+  }
+
   // Append comments
   const modelComments = [
     ...modelFrom.comments,

src/services/sync/dependencyGraph.ts

@@ -125,6 +125,13 @@ export function extractFrameworkDependencies(
     }
   }
 
+  // Authored forced refs (--depends_on comments) also affect sync order.
+  if ('depends_on' in modelJson && modelJson.depends_on) {
+    for (const modelName of modelJson.depends_on) {
+      dependencies.add(modelName);
+    }
+  }
+
   return Array.from(dependencies);
 }
 

src/shared/schema/types/model.depends_on.schema.d.ts

@@ -0,0 +1,17 @@
+/* eslint-disable */
+/**
+ * This file was automatically generated by json-schema-to-typescript.
+ * DO NOT MODIFY IT BY HAND. Instead, modify the source JSONSchema file,
+ * and run json-schema-to-typescript to regenerate this file.
+ */
+
+/**
+ * Additional model names whose ref() edges should be forced in generated SQL via --depends_on comments (for refs hidden from dbt parse, e.g. inside {% if execute %})
+ *
+ * @minItems 1
+ */
+export type SchemaModelDependsOn = [SchemaModelRef, ...SchemaModelRef[]];
+/**
+ * Validate model ids
+ */
+export type SchemaModelRef = string;