Skip to content

Commit

Permalink
Fix compile_ibis_to_sql_models function. Add option to pass parse arg…
Browse files Browse the repository at this point in the history
…uments (#21)
  • Loading branch information
binste authored Oct 4, 2023
1 parent fab97d4 commit f0b7a9f
Show file tree
Hide file tree
Showing 3 changed files with 33 additions and 22 deletions.
35 changes: 20 additions & 15 deletions dbt_ibis/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -253,10 +253,10 @@ def _do_nothing(*args: Any, **kwargs: Any) -> None: # noqa: ARG001
manifest.invalid_target_fail_unless_test = original_func


def compile_ibis_to_sql_models() -> None:
def compile_ibis_to_sql_models(dbt_parse_arguments: Optional[list[str]] = None) -> None:
logger.info("Parse dbt project")
with _disable_node_not_found_error():
manifest, runtime_config = _invoke_parse_customized()
manifest, runtime_config = _invoke_parse_customized(dbt_parse_arguments)

ibis_dialect = _dialects.get_ibis_dialect(manifest)

Expand Down Expand Up @@ -311,8 +311,20 @@ def compile_ibis_to_sql_models() -> None:
logger.info("Finished compiling Ibis models to SQL")


def _invoke_parse_customized() -> tuple[Manifest, RuntimeConfig]:
args = _get_parse_arguments()
def _invoke_parse_customized(
dbt_parse_arguments: Optional[list[str]],
) -> tuple[Manifest, RuntimeConfig]:
dbt_parse_arguments = dbt_parse_arguments or []
parse_command = _parse_customized.name
# For the benefit of mypy
assert isinstance(parse_command, str) # noqa: S101
# Use --quiet to suppress non-error logs in stdout. These logs would be
# confusing to a user as they don't expect two dbt commands to be executed.
# Furthermore, the logs might contain warnings which the user can ignore
# as they come from the fact that Ibis models might not yet be present as .sql
# files when running the parse command.
args = ["--quiet", parse_command, *dbt_parse_arguments]

dbt_ctx = cli.make_context(cli.name, args)
result, success = cli.invoke(dbt_ctx)
if not success:
Expand All @@ -326,23 +338,15 @@ def _get_parse_arguments() -> list[str]:
# any global flags that come before it. All subsequent arguments are passed to
# _parse_customized so that a user can e.g. set --project-dir etc.
# For example, "dbt-ibis --warn-error run --select stg_orders --project-dir folder"
# becomes "parse_customized run --select stg_orders --project-dir folder"
# becomes "--select stg_orders --project-dir folder"
# in variable args. parse_customized will then ignore "--select stg_orders"
all_args = sys.argv[1:]
subcommand_idx = next(
i
for i, arg in enumerate(all_args)
if arg in [*list(cli.commands.keys()), "precompile"]
)
parse_command = _parse_customized.name
# For the benefit of mypy
assert isinstance(parse_command, str) # noqa: S101
# Use --quiet to suppress non-error logs in stdout. These logs would be
# confusing to a user as they don't expect two dbt commands to be executed.
# Furthermore, the logs might contain warnings which the user can ignore
# as they come from the fact that Ibis models might not yet be present as .sql
# files when running the parse command.
args = ["--quiet", parse_command] + all_args[subcommand_idx + 1 :]
args = all_args[subcommand_idx + 1 :]
return args


Expand Down Expand Up @@ -540,7 +544,8 @@ def _to_dbt_sql(


def main() -> None:
compile_ibis_to_sql_models()
dbt_parse_arguments = _get_parse_arguments()
compile_ibis_to_sql_models(dbt_parse_arguments)
# Rudimentary approach to adding a "precompile" command to dbt-ibis.
# If there are any global flags before precompile, this would fail
if sys.argv[1] != "precompile":
Expand Down
2 changes: 1 addition & 1 deletion demo_project/jaffle_shop/models/staging/stg_payments.ibis
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ from dbt_ibis import depends_on, ref
@depends_on(ref("raw_payments"))
def model(raw_payments):
raw_payments = (
raw_payments.relabel({"id": "payment_id"})
raw_payments.rename(payment_id="id")
.mutate(
# `amount` is currently stored in cents, so we convert it to dollars
amount=d_["amount"]
Expand Down
18 changes: 12 additions & 6 deletions tests/test_dbt_ibis.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
_IbisModel,
_sort_ibis_models_by_dependencies,
_to_dbt_sql,
compile_ibis_to_sql_models,
depends_on,
ref,
source,
Expand Down Expand Up @@ -448,8 +449,6 @@ def test_get_parse_arguments(mocker):
args = _get_parse_arguments()

assert args == [
"--quiet",
"parse_customized",
"--project-dir",
"some_folder",
"--select",
Expand All @@ -467,10 +466,7 @@ def test_get_parse_arguments(mocker):

args = _get_parse_arguments()

assert args == [
"--quiet",
"parse_customized",
]
assert args == []


def execute_command(cmd: list[str]) -> None:
Expand Down Expand Up @@ -525,6 +521,16 @@ def test_precompile_command(project_dir_and_database_file: tuple[Path, Path]):
validate_compiled_sql_files(project_dir)


def test_compile_ibis_to_sql_models(project_dir_and_database_file: tuple[Path, Path]):
project_dir, database_file = project_dir_and_database_file
compile_ibis_to_sql_models()

assert (
not database_file.exists()
), "Database was created although precompile command should not create it."
validate_compiled_sql_files(project_dir)


def test_end_to_end(project_dir_and_database_file: tuple[Path, Path]):
project_dir, database_file = project_dir_and_database_file

Expand Down

0 comments on commit f0b7a9f

Please sign in to comment.