Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,42 @@ impl QueryDateTime {
)
}

/// Whether this instant sits exactly on the start of the given predefined granularity —
/// i.e. whether `DATE_TRUNC(granularity, self)` would leave it unchanged.
///
/// Judged on the local wall clock, so a midnight that a DST gap pushed forward to 01:00 still
/// counts as the start of its day.
pub fn is_start_of(&self, granularity: &str) -> Result<bool, CubeError> {
let dt = self.naive_local();
// Midnight can be absent from the local calendar when a DST gap swallows it; the day then
// starts at whatever instant the gap resolves to, and that is still its start.
let midnight = dt.date().and_hms_opt(0, 0, 0).ok_or_else(|| {
CubeError::internal(format!("Failed to build midnight for {}", dt.date()))
})?;
Comment on lines +66 to +68

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

NaiveDate::and_hms_opt(0, 0, 0) never returns None — midnight is representable for every NaiveDate — so this error arm is unreachable. dt.date().and_time(NaiveTime::MIN) gives the same value without the Result plumbing, which also lets the fallible part of the function be just from_local_date_time.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed, and_hms_opt(0, 0, 0) can't fail. Leaving it for the same push that deals with the date_bin boundary problem above rather than force-pushing a cosmetic change while that's still open.

let start_of_day =
Self::from_local_date_time(self.date_time.timezone(), midnight)?.naive_local();
let starts_the_day = dt.time() == start_of_day.time() && dt.date() == start_of_day.date();
let zero_time = starts_the_day && dt.nanosecond() == 0;

let res = match granularity {
"year" => zero_time && dt.month() == 1 && dt.day() == 1,
"quarter" => zero_time && dt.day() == 1 && (dt.month() - 1).is_multiple_of(3),
"month" => zero_time && dt.day() == 1,
"week" => zero_time && dt.weekday() == Weekday::Mon,
"day" => zero_time,
"hour" => dt.minute() == 0 && dt.second() == 0 && dt.nanosecond() == 0,
"minute" => dt.second() == 0 && dt.nanosecond() == 0,
"second" => dt.nanosecond() == 0,
other => {
return Err(CubeError::internal(format!(
"Unexpected granularity '{other}' in date alignment check"
)))
}
};
Comment on lines +74 to +88

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This match is a near-duplicate of QueryDateTime::granularity() (lines 185-210) — same "is this instant on a boundary" question, but that one has no DST tolerance and no quarter arm, and it cascades instead of answering per unit.

The two now co-decide the same grain: for the Asuncion-style origin from your test, is_start_of("month") is true (so apply_to_input_sql emits DATE_TRUNC('month', …)), while Granularity::min_granularity()origin.granularity() returns "hour" because local time is 01:00 — which feeds pre-aggregation matching. That inconsistency predates this PR, but the new method makes it easy to remove: expressing granularity() as a descending scan over is_start_of would keep the two from drifting and pick up the DST handling for free.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fair, and the inconsistency you name is real: is_start_of("month") accepts the Asuncion DST-gap origin while granularity() reports hour for the same instant, and that one feeds pre-aggregation matching. Worth collapsing granularity() into a descending scan over is_start_of, but it changes pre-agg matching behaviour, so not as a rider on this fix.


Ok(res)
}

pub fn date_time(&self) -> DateTime<Tz> {
self.date_time
}
Expand Down Expand Up @@ -205,6 +241,29 @@ impl QueryDateTime {
mod tests {
use super::*;

#[test]
fn is_start_of_accepts_a_midnight_swallowed_by_a_dst_gap() {
// Paraguay springs forward at midnight, so 2023-10-01T00:00 does not exist locally and
// resolves to 01:00. It is still the first instant of that day, month and quarter.
let tz = "America/Asuncion".parse::<Tz>().unwrap();
let d = QueryDateTime::from_date_str(tz, "2023-10-01").unwrap();

assert_eq!(d.default_format(), "2023-10-01T01:00:00.000");
assert!(d.is_start_of("day").unwrap());
assert!(d.is_start_of("month").unwrap());
assert!(d.is_start_of("quarter").unwrap());
assert!(!d.is_start_of("year").unwrap());
}

#[test]
fn is_start_of_still_rejects_a_real_time_component() {
let tz = "America/Asuncion".parse::<Tz>().unwrap();
// 2023-10-02 has an ordinary midnight, so 01:00 on it is genuinely mid-day.
let d = QueryDateTime::from_date_str(tz, "2023-10-02T01:00:00").unwrap();
assert!(!d.is_start_of("day").unwrap());
assert!(d.is_start_of("hour").unwrap());
}

#[test]
fn test_parse_date_time() {
let tz = "Etc/GMT-3".parse::<Tz>().unwrap();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,24 +67,38 @@ impl Granularity {
});
}

let origin = if let Some(origin) = origin {
QueryDateTime::from_date_str(timezone, &origin)?
// `origin` takes precedence over `offset`, and drops it: keeping the offset around would
// let the rendering shift buckets that `origin` — which every alignment check reads —
// knows nothing about.
let (origin, granularity_offset) = if let Some(origin) = origin {
(QueryDateTime::from_date_str(timezone, &origin)?, None)
} else if let Some(offset) = &granularity_offset {
// Week-based intervals expect the offset relative to the start of a week.
let origin = Self::fix_origin_for_weeks_if_needed(
Self::default_origin(timezone)?,
&granularity_interval,
);
let interval = SqlInterval::from_str(offset)?;
origin.add_interval(&interval)?
(origin.add_interval(&interval)?, granularity_offset)
} else {
Self::fix_origin_for_weeks_if_needed(
Self::default_origin(timezone)?,
&granularity_interval,
(
Self::fix_origin_for_weeks_if_needed(
Self::default_origin(timezone)?,
&granularity_interval,
),
granularity_offset,
)
};

let is_natural_aligned = granularity_interval.is_trivial();
// A trivial interval can only ride the DATE_TRUNC path when its origin actually sits on
// that unit's natural boundary — `1 year` from 2024-04-01 is a fiscal year, and truncating
// to the calendar year would silently discard the origin. An `offset` origin is exempt:
// it is off-boundary by construction, and the aligned branch renders it by subtracting the
// offset before truncating and adding it back.
let is_natural_aligned = granularity_interval.is_trivial()
&& (granularity_offset.is_some()
|| origin.is_start_of(&granularity_interval.min_granularity()?)?);
Comment thread
igorlukanin marked this conversation as resolved.

Ok(Self {
granularity,
granularity_interval,
Expand Down Expand Up @@ -293,4 +307,74 @@ mod tests {
NaiveDate::from_ymd_opt(2024, 1, 3).unwrap()
);
}

#[test]
fn trivial_interval_with_off_boundary_origin_is_not_natural_aligned() {
// A fiscal year starting on April 1 cannot be rendered as DATE_TRUNC('year', ...).
assert!(!custom("1 year", Some("2024-04-01"), None).is_natural_aligned());
}

#[test]
fn trivial_interval_with_on_boundary_origin_stays_natural_aligned() {
// January 1 is the natural year boundary, so DATE_TRUNC is still correct.
assert!(custom("1 year", Some("2024-01-01"), None).is_natural_aligned());
}

#[test]
fn off_boundary_origin_is_judged_against_the_intervals_own_unit() {
// April 1 is off the year boundary but exactly on the month and quarter boundaries.
assert!(custom("1 month", Some("2024-04-01"), None).is_natural_aligned());
assert!(custom("1 quarter", Some("2024-04-01"), None).is_natural_aligned());
// ...and February 1 is on the month boundary but not the quarter one.
assert!(custom("1 month", Some("2024-02-01"), None).is_natural_aligned());
assert!(!custom("1 quarter", Some("2024-02-01"), None).is_natural_aligned());
}

#[test]
fn origin_with_a_time_component_defeats_alignment_of_date_intervals() {
assert!(!custom("1 day", Some("2024-04-01T06:00:00"), None).is_natural_aligned());
assert!(custom("1 hour", Some("2024-04-01T06:00:00"), None).is_natural_aligned());
}

#[test]
fn week_origin_is_judged_against_monday() {
// 2024-01-01 is a Monday; 2024-01-03 is a Wednesday.
assert!(custom("1 week", Some("2024-01-01"), None).is_natural_aligned());
assert!(!custom("1 week", Some("2024-01-03"), None).is_natural_aligned());
}

#[test]
fn default_and_offset_origins_keep_their_existing_alignment() {
// No explicit origin: the default origin is the start of the year, and the week-only
// interval snaps to Monday — both natural boundaries, so DATE_TRUNC still applies.
assert!(custom("1 year", None, None).is_natural_aligned());
assert!(custom("1 week", None, None).is_natural_aligned());
// `offset` renders through the subtract/truncate/add branch, which stays aligned.
assert!(custom("1 week", None, Some("-1 day")).is_natural_aligned());
}

#[test]
fn explicit_origin_discards_the_offset() {
// `origin` takes precedence when both are set. The offset must be dropped outright —
// left in place it would shift the rendered buckets away from the origin that
// `align_date_to_origin` and the materialized time series both bin on.
let g = custom("1 year", Some("2024-04-01"), Some("3 months"));
assert_eq!(g.granularity_offset(), &None);
assert!(!g.is_natural_aligned());

// Same for an on-boundary origin, which would otherwise stay on the offset sub-branch.
let g = custom("1 year", Some("2024-01-01"), Some("3 months"));
assert_eq!(g.granularity_offset(), &None);
assert_eq!(
origin_date(&g),
NaiveDate::from_ymd_opt(2024, 1, 1).unwrap()
);
assert!(g.is_natural_aligned());
}

#[test]
fn non_trivial_intervals_are_never_natural_aligned() {
assert!(!custom("15 minutes", None, None).is_natural_aligned());
assert!(!custom("6 months", Some("2024-01-01"), None).is_natural_aligned());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,14 @@ cubes:
- name: fiscal_year
interval: "1 year"
offset: "1 month"
# Trivial interval with an off-boundary origin: must render through DATE_BIN,
# not DATE_TRUNC, or the origin is silently discarded.
- name: fiscal_year_by_1st_april
interval: "1 year"
origin: "2024-04-01"
- name: calendar_year_by_origin
interval: "1 year"
origin: "2024-01-01"

- name: fiscal_year_alias
type: time
Expand All @@ -39,3 +47,11 @@ cubes:
- name: total_amount
type: sum
sql: amount
- name: total_amount_prior_year
type: number
sql: "{CUBE.total_amount}"
multi_stage: true
time_shift:
- time_dimension: created_at
interval: "1 year"
type: prior
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,100 @@ async fn test_type_time_alias_wraps_compound_exprs_before_tz_cast() {
}
}

#[tokio::test(flavor = "multi_thread")]
async fn test_fiscal_year_by_origin_does_not_truncate_to_calendar_year() {
let ctx = create_context();

let query = indoc! {"
measures:
- orders.count
time_dimensions:
- dimension: orders.created_at
granularity: fiscal_year_by_1st_april
dateRange:
- \"2024-04-01\"
- \"2026-03-31\"
order:
- id: orders.created_at
"};

let sql = ctx.build_sql(query).unwrap();

// `1 year` is a trivial interval, so the grain used to fall onto the DATE_TRUNC branch,
// which has no way to express an April 1 origin.
assert!(
!sql.to_lowercase().contains("date_trunc('year'"),
"off-boundary origin was truncated to the calendar year\nFull SQL:\n{sql}"
);

if let Some(result) = ctx.try_execute_pg(query, SEED).await {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The try_execute_pg half of these tests was never exercised — and it's the half that would catch the remaining problem.

No .snap files are committed for test_fiscal_year_by_origin_does_not_truncate_to_calendar_year, test_calendar_year_by_origin_still_truncates, or test_fiscal_year_by_origin_with_time_shift_measure, while the other tests in this file have them (e.g. …custom_granularities__fiscal_year_with_offset.snap). Under --features integration-postgres these three will fail on missing snapshots; without the feature try_execute_pg returns None and the assertion is skipped entirely. So the only thing verified is "the string date_trunc('year' is absent".

That matters because the DATE_BIN branch this now routes to is epoch-approximated on Postgres:

('2024-04-01'::timestamp + INTERVAL '1 year' *
  FLOOR(EXTRACT(EPOCH FROM (src - '2024-04-01'::timestamp)) / EXTRACT(EPOCH FROM INTERVAL '1 year')))

EXTRACT(EPOCH FROM INTERVAL '1 year') is 31 557 600 (365.25 d). A row on 2025-04-01 is 365 days past the origin → FLOOR(31536000 / 31557600) = 0 → it lands in the 2024-04-01 bucket, i.e. the fiscal-year boundary is off by a day, and it drifts differently each year within the leap cycle. 1 month is worse: epoch of INTERVAL '1 month' is 30 days, so origin 2025-01-15 puts 2025-03-15 (59 days) into the February bucket.

Meanwhile the planner's own bucketing — align_date_to_originQueryDateTime::add_interval — is calendar-accurate, so the SQL buckets and the materialized/generated time series can disagree at exactly these boundaries.

Running these tests against Postgres and committing the snapshots would either confirm the fiscal-year buckets are right or surface the drift; right now the PR's headline example isn't verified end to end.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You're right, and it's worse than the snapshots being absent. Working it through against the CI run that did execute these (unit-core on b921f35, which failed only on the missing .snap files):

The fixture rows bucket correctly — 2024-04-01 | 8, 2025-04-01 | 4 — because none of them lands on a fiscal-year boundary. Add a row dated exactly 2025-04-01 and it files into the 2024-04-01 bucket: EXTRACT(EPOCH FROM INTERVAL '1 year') is 31 557 600 (365.25 d), the span from 2024-04-01 to 2025-04-01 is 365 d = 31 536 000 s, so FLOOR gives 0. For origin 2024-04-01 that's wrong on the +1, +2, +3, +5, +6 year boundaries and right only on +4.

1 year is the only unit that misbehaves here, which is why this wasn't already visible: INTERVAL '1 month' (30 d) and '6 months' (180 d) both under-count real calendar time, so FLOOR never overshoots and the existing 6 months + origin grains are stable. '1 year' is the one that over-counts — and it's exactly the unit this PR newly routes onto date_bin.

Meanwhile align_date_to_origin bins with calendar-accurate add_interval, so the SQL buckets and the generated time series disagree at precisely those boundaries. Snowflake is fine (DATEDIFF(year, …) is calendar-exact) and Redshift already overrides dateBin for this reason; it's Postgres and Crate that are exposed.

Fixing it properly means giving PostgresQuery.dateBin the month-arithmetic treatment RedshiftQuery already has, which is a change to shared JS SQL generation for every Postgres-family dialect — outside this PR's Tesseract-only scope and with its own backward-compat surface for the existing 6 months grains. Escalating rather than folding it in; not committing the snapshots in the meantime, since they'd bless output that's wrong one row over.

insta::assert_snapshot!(result);
}
}

#[tokio::test(flavor = "multi_thread")]
async fn test_calendar_year_by_origin_still_truncates() {
let ctx = create_context();

let query = indoc! {"
measures:
- orders.count
time_dimensions:
- dimension: orders.created_at
granularity: calendar_year_by_origin
dateRange:
- \"2024-01-01\"
- \"2025-12-31\"
order:
- id: orders.created_at
"};

let sql = ctx.build_sql(query).unwrap();

// An origin that already sits on the year boundary keeps the cheaper DATE_TRUNC rendering.
assert!(
sql.to_lowercase().contains("date_trunc('year'"),
"on-boundary origin should still truncate\nFull SQL:\n{sql}"
);

if let Some(result) = ctx.try_execute_pg(query, SEED).await {
insta::assert_snapshot!(result);
}
}

#[tokio::test(flavor = "multi_thread")]
async fn test_fiscal_year_by_origin_with_time_shift_measure() {
let ctx = create_context();

// The shape from the original report: a time-shift measure alongside the plain one, both
// grouped by a fiscal-year grain. The shifted leaf must bin on the same origin as the
// unshifted one, or the two sides join on incompatible buckets.
let query = indoc! {"
measures:
- orders.total_amount
- orders.total_amount_prior_year
time_dimensions:
- dimension: orders.created_at
granularity: fiscal_year_by_1st_april
dateRange:
- \"2024-04-01\"
- \"2026-03-31\"
order:
- id: orders.created_at
"};

let sql = ctx.build_sql(query).unwrap();

assert!(
!sql.to_lowercase().contains("date_trunc('year'"),
"time-shifted leaf truncated to the calendar year\nFull SQL:\n{sql}"
);

if let Some(result) = ctx.try_execute_pg(query, SEED).await {
insta::assert_snapshot!(result);
}
}

#[tokio::test(flavor = "multi_thread")]
async fn test_custom_granularity_with_daterange_filter() {
let ctx = create_context();
Expand Down
Loading