Skip to content

Commit 7a3f5f5

Browse files
fix: accept dbt 2.0 capitalized source freshness statuses in sources v3 parser
dbt 2.0.0-preview.202 emits 'Pass'/'Error' where earlier dbt emitted lowercase. Every freshness result then failed validation and the whole sources.json was silently dropped (live: harvestgroup, 50 runs/day). Same class + same fix pattern as run_results_v6 'reused' (AI-7435/#106): str Enum + _missing_. Known statuses canonicalize by case to their lowercase members; unknown future statuses become dynamic members. The runtime-error union arm canonicalizes case only and stays narrow so it cannot swallow freshness rows. Version 0.3.5 -> 0.3.6. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent 10f67a9 commit 7a3f5f5

6 files changed

Lines changed: 131 additions & 6 deletions

File tree

.bumpversion.cfg

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
[bumpversion]
2-
current_version = 0.3.5
2+
current_version = 0.3.6
33
commit = True
44
tag = True
55

docs/conf.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
year = "2024"
1616
author = "Altimate Inc."
1717
copyright = f"{year}, {author}"
18-
version = release = "0.3.5"
18+
version = release = "0.3.6"
1919

2020
pygments_style = "trac"
2121
templates_path = ["."]

setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ def read(*names, **kwargs):
1313

1414
setup(
1515
name="altimate-datapilot-cli",
16-
version="0.3.5",
16+
version="0.3.6",
1717
license="MIT",
1818
description="Assistant for Data Teams",
1919
long_description="{}\n{}".format(

src/datapilot/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
__version__ = "0.3.5"
1+
__version__ = "0.3.6"

src/vendor/dbt_artifacts_parser/parsers/sources/sources_v3.py

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,23 @@ class Metadata(BaseParserModel):
2525
env: Optional[dict[str, str]] = None
2626

2727

28-
class Status(Enum):
28+
class Status(str, Enum):
2929
runtime_error = "runtime error"
3030

31+
@classmethod
32+
def _missing_(cls, value):
33+
# dbt 2.0 capitalizes freshness statuses (e.g. "Runtime Error").
34+
# Canonicalize case here, but stay NARROW otherwise: this enum is the
35+
# runtime-error union arm and must not swallow freshness rows (see
36+
# Status1 below for the permissive arm). Same class of fix as
37+
# run_results_v6.Status (AI-7435 / PR #106).
38+
if isinstance(value, str):
39+
low = value.lower()
40+
for member in cls:
41+
if member.value == low:
42+
return member
43+
return None
44+
3145

3246
class Results(BaseParserModel):
3347
model_config = ConfigDict(
@@ -38,12 +52,31 @@ class Results(BaseParserModel):
3852
status: Status
3953

4054

41-
class Status1(Enum):
55+
class Status1(str, Enum):
4256
pass_ = "pass"
4357
warn = "warn"
4458
error = "error"
4559
runtime_error = "runtime error"
4660

61+
@classmethod
62+
def _missing_(cls, value):
63+
# dbt 2.0 (>= 2.0.0-preview.202) emits capitalized freshness statuses
64+
# ("Pass", "Error") where every earlier dbt emitted lowercase. That made
65+
# ALL source freshness results fail validation and silently dropped the
66+
# whole sources.json during ingestion. Canonicalize known values by
67+
# case; surface truly unknown future statuses as real members so
68+
# downstream `.value` access keeps working (same forward-compat pattern
69+
# as run_results_v6.Status, AI-7435 / PR #106).
70+
if isinstance(value, str):
71+
low = value.lower()
72+
for member in cls:
73+
if member.value == low:
74+
return member
75+
member = str.__new__(cls, value)
76+
member._name_ = str(value)
77+
member._value_ = value
78+
return member
79+
4780

4881
class Period(Enum):
4982
minute = "minute"
Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
"""Tests for sources v3 parser: resilient freshness Status enums.
2+
3+
Regression coverage for dbt 2.0 (>= 2.0.0-preview.202) emitting CAPITALIZED
4+
freshness statuses ("Pass", "Error") where earlier dbt emitted lowercase.
5+
Previously every result failed validation and the entire sources.json was
6+
silently dropped during ingestion (observed live: harvestgroup, 50 runs/day).
7+
Same defect class as run_results_v6 ``reused`` (AI-7435 / PR #106).
8+
"""
9+
from vendor.dbt_artifacts_parser.parser import parse_sources
10+
from vendor.dbt_artifacts_parser.parsers.sources.sources_v3 import Status
11+
from vendor.dbt_artifacts_parser.parsers.sources.sources_v3 import Status1
12+
13+
V3_SCHEMA = "https://schemas.getdbt.com/dbt/sources/v3.json"
14+
15+
16+
def _freshness_result(status: str, unique_id: str) -> dict:
17+
return {
18+
"unique_id": unique_id,
19+
"max_loaded_at": "2026-07-28T00:00:00Z",
20+
"snapshotted_at": "2026-07-28T01:00:00Z",
21+
"max_loaded_at_time_ago_in_s": 3600.0,
22+
"status": status,
23+
"criteria": {
24+
"warn_after": {"count": 12, "period": "hour"},
25+
"error_after": {"count": 24, "period": "hour"},
26+
},
27+
"adapter_response": {},
28+
"timing": [],
29+
"thread_id": "Thread-1",
30+
"execution_time": 1.0,
31+
}
32+
33+
34+
def _sources(results: list) -> dict:
35+
return {
36+
"metadata": {
37+
"dbt_schema_version": V3_SCHEMA,
38+
"dbt_version": "2.0.0-preview.202",
39+
"generated_at": "2026-07-28T01:00:00Z",
40+
"invocation_id": "11111111-1111-1111-1111-111111111111",
41+
"env": {},
42+
},
43+
"results": results,
44+
"elapsed_time": 2.0,
45+
}
46+
47+
48+
def test_capitalized_pass_and_error_parse_and_canonicalize():
49+
"""The exact live failure: dbt 2.0 'Pass'/'Error' must parse, and known
50+
statuses canonicalize to their lowercase members."""
51+
parsed = parse_sources(
52+
_sources(
53+
[
54+
_freshness_result("Pass", "source.p.s.a"),
55+
_freshness_result("Error", "source.p.s.b"),
56+
]
57+
)
58+
)
59+
statuses = [r.status for r in parsed.results]
60+
assert statuses[0] is Status1.pass_
61+
assert statuses[0].value == "pass"
62+
assert statuses[1] is Status1.error
63+
assert statuses[1].value == "error"
64+
65+
66+
def test_lowercase_statuses_still_parse_unchanged():
67+
parsed = parse_sources(
68+
_sources(
69+
[
70+
_freshness_result("pass", "source.p.s.a"),
71+
_freshness_result("warn", "source.p.s.b"),
72+
_freshness_result("error", "source.p.s.c"),
73+
]
74+
)
75+
)
76+
assert [r.status.value for r in parsed.results] == ["pass", "warn", "error"]
77+
78+
79+
def test_unknown_future_status_surfaces_as_member():
80+
"""Forward-compat: a status dbt invents later must not drop the file."""
81+
parsed = parse_sources(_sources([_freshness_result("Stale", "source.p.s.a")]))
82+
assert parsed.results[0].status.value == "Stale"
83+
84+
85+
def test_runtime_error_arm_canonicalizes_case_but_stays_narrow():
86+
"""The runtime-error union arm accepts case variants of its one value and
87+
nothing else — it must not swallow freshness rows."""
88+
assert Status("Runtime Error") is Status.runtime_error
89+
assert Status1("Runtime Error") is Status1.runtime_error
90+
runtime_row = {"unique_id": "source.p.s.x", "status": "runtime error", "error": "boom"}
91+
parsed = parse_sources(_sources([runtime_row]))
92+
assert parsed.results[0].status.value == "runtime error"

0 commit comments

Comments
 (0)