More fixes

Author: shaidar

dg_projects/edxorg/Dockerfile

@@ -15,6 +15,7 @@ WORKDIR /app
 # Copy the entire workspace structure needed for uv
 COPY packages/ol-orchestrate-lib packages/ol-orchestrate-lib
 COPY dg_projects/edxorg dg_projects/edxorg
+COPY src/ol_dbt src/ol_dbt
 
 # Install dependencies from the edxorg project
 WORKDIR /app/dg_projects/edxorg
@@ -26,6 +27,12 @@ RUN --mount=type=cache,target=/root/.cache/uv \
 RUN --mount=type=cache,target=/root/.cache/uv \
     uv sync --frozen --no-dev --no-editable
 
+# Compile dbt to generate manifest.json (CRITICAL for @dbt_assets)
+WORKDIR /app/src/ol_dbt
+RUN /app/dg_projects/edxorg/.venv/bin/dbt deps && \
+    /app/dg_projects/edxorg/.venv/bin/dbt compile --target production || \
+    /app/dg_projects/edxorg/.venv/bin/dbt compile --target dev
+
 
 # Then, use a final image without uv
 FROM python:3.13-slim-bookworm
@@ -34,6 +41,7 @@ FROM python:3.13-slim-bookworm
 
 # Copy the application from the builder
 COPY --from=builder /app/dg_projects/edxorg /app
+COPY --from=builder /app/src/ol_dbt /app/ol_dbt
 
 # Place executables in the environment at the front of the path
 ENV PATH="/app/.venv/bin:$PATH"

dg_projects/edxorg/edxorg/definitions.py

@@ -8,6 +8,7 @@
 
 import os
 from functools import partial
+from pathlib import Path
 from typing import Any, Literal
 
 from dagster import (
@@ -21,15 +22,12 @@
     job,
 )
 from dagster_aws.s3 import S3Resource
-from dagster_dbt import DbtCliResource, dbt_assets
+from dagster_dbt import DbtCliResource, DbtProject, dbt_assets
 from ol_orchestrate.io_managers.filepath import (
     FileObjectIOManager,
     S3FileObjectIOManager,
 )
 from ol_orchestrate.lib.constants import DAGSTER_ENV, VAULT_ADDRESS
-
-# Import dbt project
-from ol_orchestrate.lib.dbt_project import dbt_project
 from ol_orchestrate.lib.utils import authenticate_vault
 from ol_orchestrate.resources.gcp_gcs import GCSConnection
 from ol_orchestrate.resources.github import GithubApiClientFactory
@@ -69,6 +67,16 @@
     upload_files_to_s3,
 )
 
+# Initialize dbt project - handle both local dev and Docker paths
+if Path("/app/ol_dbt").exists():
+    # In Docker container
+    DBT_PROJECT_DIR = Path("/app/ol_dbt")
+else:
+    # Local development
+    DBT_PROJECT_DIR = Path(__file__).resolve().parents[3] / "src" / "ol_dbt"
+
+dbt_project = DbtProject(project_dir=DBT_PROJECT_DIR)
+
 # Initialize vault with resilient loading
 try:
     vault = authenticate_vault(DAGSTER_ENV, VAULT_ADDRESS)
@@ -288,7 +296,7 @@ def edxorg_dbt_assets(context: AssetExecutionContext, dbt: DbtCliResource):
         sync_edxorg_program_reports,
     ],
     assets=[
-        edxorg_dbt_assets,
+        edxorg_dbt_assets,  # Re-enable dbt assets
         edxorg_raw_data_archive.to_source_asset(),
         edxorg_raw_tracking_logs.to_source_asset(),
         normalize_edxorg_tracking_log,