Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Support Ibis 9. Use Ibis-native datatype parsing for BigQuery. … #55

Merged
merged 2 commits into from
May 2, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions RELEASING.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
3. Commit change and push:

git add . -u
git commit -m "MAINT: Bump version to 0.2.0"
git commit -m "maint: Bump version to 0.2.0"
git push

4. Merge release branch into main, make sure that all required checks pass
Expand Down Expand Up @@ -47,7 +47,7 @@
12. Commit change and push:

git add . -u
git commit -m "MAINT: Bump version to 0.3.0dev"
git commit -m "maint: Bump version to 0.3.0dev"
git push

13. Merge maintenance branch into main
2 changes: 1 addition & 1 deletion dbt_ibis/__init__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
__all__ = ["ref", "source", "depends_on", "compile_ibis_to_sql"]
__version__ = "0.9.0dev"
__version__ = "0.9.0"

import logging
import subprocess
Expand Down
28 changes: 9 additions & 19 deletions dbt_ibis/_dialects.py
Original file line number Diff line number Diff line change
@@ -1,22 +1,12 @@
from typing import NewType

import ibis
import ibis.backends.base.sqlglot.datatypes as sqlglot_dt
import ibis.backends.sql.datatypes as sql_dt
import ibis.expr.datatypes as dt
import ibis.expr.types as ir
from dbt.contracts.graph.manifest import Manifest
from ibis.formats import TypeMapper


# Custom BigQuery type until a corresponding one is implemented in Ibis itself.
# See https://github.com/ibis-project/ibis/issues/7531
class BigQueryType(sqlglot_dt.SqlglotType):
dialect = "bigquery"

default_decimal_precision = 38
default_decimal_scale = 9


# Use NewType to make sure that we don't accidentally mix these up, i.e.
# pass a DBTAdapterType to a function that expects an IbisDialect or vice versa.
IbisDialect = NewType("IbisDialect", str)
Expand All @@ -35,14 +25,14 @@ class BigQueryType(sqlglot_dt.SqlglotType):
}

IbisDialectToTypeMapper: dict[IbisDialect, type[TypeMapper]] = {
IbisDialect("postgres"): sqlglot_dt.PostgresType,
IbisDialect("snowflake"): sqlglot_dt.SnowflakeType,
IbisDialect("trino"): sqlglot_dt.TrinoType,
IbisDialect("mysql"): sqlglot_dt.MySQLType,
IbisDialect("sqlite"): sqlglot_dt.SQLiteType,
IbisDialect("oracle"): sqlglot_dt.OracleType,
IbisDialect("duckdb"): sqlglot_dt.DuckDBType,
IbisDialect("bigquery"): BigQueryType,
IbisDialect("postgres"): sql_dt.PostgresType,
IbisDialect("snowflake"): sql_dt.SnowflakeType,
IbisDialect("trino"): sql_dt.TrinoType,
IbisDialect("mysql"): sql_dt.MySQLType,
IbisDialect("sqlite"): sql_dt.SQLiteType,
IbisDialect("oracle"): sql_dt.OracleType,
IbisDialect("duckdb"): sql_dt.DuckDBType,
IbisDialect("bigquery"): sql_dt.BigQueryType,
}


Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ build-backend = "hatchling.build"
[project]
name = "dbt-ibis"
authors = [{ name = "Stefan Binder" }]
dependencies = ["dbt-core>=1.5", "ibis-framework>=7", "click"]
dependencies = ["dbt-core>=1.5", "ibis-framework>=9", "click"]
readme = "README.md"
requires-python = ">=3.9"
dynamic = ["version"]
Expand Down
32 changes: 15 additions & 17 deletions tests/test_dbt_ibis.py
Original file line number Diff line number Diff line change
Expand Up @@ -413,14 +413,14 @@ def test_to_dbt_sql():

assert (
dbt_sql
== """\
== '''\
SELECT
t0.order_id,
t0.customer_id,
t1.customer_id AS customer_id_right
FROM {{ source('source1', 'orders') }} AS t0
LEFT OUTER JOIN {{ ref('stg_customers') }} AS t1
ON t0.customer_id = t1.customer_id"""
"t2"."order_id",
"t2"."customer_id",
"t3"."customer_id" AS "customer_id_right"
FROM {{ source('source1', 'orders') }} AS "t2"
LEFT OUTER JOIN {{ ref('stg_customers') }} AS "t3"
ON "t2"."customer_id" = "t3"."customer_id"'''
)


Expand Down Expand Up @@ -540,12 +540,12 @@ def validate_compiled_sql_files(project_dir: Path) -> list[Path]:
stg_stores = next(p for p in compiled_sql_files if p.stem == "stg_stores")
assert (
stg_stores.read_text()
== """\
== '''\
SELECT
CAST(t0.store_id AS BIGINT) AS store_id,
t0.store_name,
t0.country
FROM {{ source('sources_db', 'stores') }} AS t0"""
CAST("t0"."store_id" AS BIGINT) AS "store_id",
"t0"."store_name",
"t0"."country"
FROM {{ source('sources_db', 'stores') }} AS "t0"'''
)

usa_stores = stg_stores = next(
Expand All @@ -555,12 +555,10 @@ def validate_compiled_sql_files(project_dir: Path) -> list[Path]:
usa_stores.read_text()
== """\
SELECT
t0.store_id,
t0.store_name,
t0.country
FROM {{ ref('stg_stores') }} AS t0
*
FROM {{ ref('stg_stores') }} AS "t0"
WHERE
t0.country = 'USA'"""
"t0"."country" = 'USA'"""
)
return compiled_sql_files

Expand Down
Loading