diff --git a/be/src/formats/orc/orc_min_max_decoder.cpp b/be/src/formats/orc/orc_min_max_decoder.cpp index fca91ad9899d98..a94162db0ba330 100644 --- a/be/src/formats/orc/orc_min_max_decoder.cpp +++ b/be/src/formats/orc/orc_min_max_decoder.cpp @@ -117,6 +117,55 @@ static Status decode_date_min_max(const orc::proto::ColumnStatistics& colStats, // It's quite odd that, timestamp statistics stores milliseconds since unix epoch time. // but timestamp column vector batch stores seconds since unix epoch time. // https://orc.apache.org/specification/ORCv1/ +// +// ORC serializes a TIMESTAMP min/max as whole milliseconds since the Unix epoch (minimumUtc / +// maximumUtc) plus a sub-millisecond nanos remainder in [0, 999999] (minimumNanos / maximumNanos). +// The nanos field is stored with a +1 offset and is omitted when equal to its default (0 for the +// minimum, 999999 for the maximum). StarRocks timestamps are microsecond precision, so each bound +// is rounded conservatively -- the minimum floored, the maximum ceiled -- to keep [min, max] a +// superset of the true value range. For a TIMESTAMP_INSTANT the reader timezone offset is folded +// into the seconds and the value is decoded as plain UTC, matching the after-epoch instant path. +// +// Note: the folded offset is the scalar reader-minus-writer offset looked up at the epoch. For a +// pre-1970 instant in a named zone whose historical offset differs from its epoch offset this is +// approximate (the row-load path uses a per-instant cctz conversion), so the bound may still +// under-cover -- a pre-existing limitation of the scalar-offset model, not specific to this path. +static void decode_orc_timestamp_bound(int64_t utc_ms, bool has_nanos, int32_t raw_nanos, bool is_max, bool is_instant, + int64_t tz_offset_in_seconds, TimestampValue* out) { + // The sub-millisecond nanos remainder is in [0, 999999]; a present field is the value + 1 (so + // in [1, 1000000]). Undo the offset, falling back to the conservative default when the field is + // absent or -- for untrusted external files -- malformed. + constexpr int64_t kMaxSubNanos = 999999; + int64_t sub_nanos = (has_nanos && raw_nanos >= 1 && raw_nanos <= kMaxSubNanos + 1) ? (raw_nanos - 1) + : (is_max ? kMaxSubNanos : 0); + // Floor the millisecond split toward -inf so the remainder (and nanos) stay non-negative. + int64_t secs = utc_ms / 1000; + int64_t rem_ms = utc_ms - secs * 1000; + if (rem_ms < 0) { + secs -= 1; + rem_ms += 1000; + } + int64_t nanos = rem_ms * NANOSECS_PER_MILLIS + sub_nanos; // nanos-of-second in [0, NANOSECS_PER_SEC) + if (is_max) { + // Ceil to microsecond so the upper bound never understates the true maximum. + int64_t us = (nanos + NANOSECS_PER_USEC - 1) / NANOSECS_PER_USEC; + if (us == USECS_PER_SEC) { + secs += 1; + nanos = 0; + } else { + nanos = us * NANOSECS_PER_USEC; + } + } + // The minimum keeps nanos: orc_ts_to_native_ts floors nanos/1000 to microseconds. + if (is_instant) { + secs += tz_offset_in_seconds; + } + // The offset is already folded in, so decode as plain UTC; the tz argument is unused on the + // is_instant=false path (the before-epoch branch hardcodes UTC, the after-epoch branch ignores it). + OrcTimestampHelper::orc_ts_to_native_ts(out, cctz::utc_time_zone(), /*tzoffset=*/0, secs, nanos, + /*is_instant=*/false); +} + static Status decode_datetime_min_max(const orc::Type* orc_type, const orc::proto::ColumnStatistics& colStats, int64_t tz_offset_in_seconds, Column* min_col, Column* max_col) { if (orc_type->getKind() != orc::TypeKind::TIMESTAMP && orc_type->getKind() != orc::TypeKind::TIMESTAMP_INSTANT) { @@ -128,50 +177,10 @@ static Status decode_datetime_min_max(const orc::Type* orc_type, const orc::prot colStats.timestampstatistics().has_maximumutc()) { const auto& stats = colStats.timestampstatistics(); TimestampValue min, max; - const cctz::time_zone utc_tzinfo = cctz::utc_time_zone(); - { - int64_t ms = stats.minimumutc(); - int64_t ns = 0; - if (stats.has_minimumnanos()) { - ns = stats.minimumnanos(); - } - int64_t secs = ms / 1000; - ns += (ms - secs * 1000) * 1000000L; - // orc_ts_to_native_ts now carries the sub-second into the before-epoch result for the - // data load path. Pre-1970 stripe-stats sub-second is a separate, pre-existing concern - // here (the ORC nanos field is stored with a +1 offset this decoder does not undo, the - // remainder above can be negative, and the before-epoch instant branch ignores the - // offset). Dropping the min sub-second does not affect pruning, but dropping the max - // sub-second understates the max bound; keep dropping it for negative-epoch bounds to - // leave pruning byte-for-byte unchanged rather than regress it. - // - // TODO: decode pre-1970 stripe-stats sub-second correctly in a dedicated stats path: - // - subtract the +1 nanos offset (mirror liborc's reader); - // - floor secs toward -inf (not toward zero) so the ms remainder stays non-negative; - // - apply the instant tz offset on the negative-epoch branch; - // - round conservatively for bounds (floor the min, ceil the max) so [min, max] still - // contains the true value range instead of understating the max. - if (secs < 0) { - ns = 0; - } - OrcTimestampHelper::orc_ts_to_native_ts(&min, utc_tzinfo, tz_offset_in_seconds, secs, ns, is_instant); - } - - { - int64_t ms = stats.maximumutc(); - int64_t ns = 0; - if (stats.has_maximumnanos()) { - ns = stats.maximumnanos(); - } - int64_t secs = ms / 1000; - ns += (ms - secs * 1000) * 1000000L; - // See the minimum branch: keep dropping the sub-second for negative-epoch bounds. - if (secs < 0) { - ns = 0; - } - OrcTimestampHelper::orc_ts_to_native_ts(&max, utc_tzinfo, tz_offset_in_seconds, secs, ns, is_instant); - } - + decode_orc_timestamp_bound(stats.minimumutc(), stats.has_minimumnanos(), stats.minimumnanos(), + /*is_max=*/false, is_instant, tz_offset_in_seconds, &min); + decode_orc_timestamp_bound(stats.maximumutc(), stats.has_maximumnanos(), stats.maximumnanos(), + /*is_max=*/true, is_instant, tz_offset_in_seconds, &max); DOWN_CAST_ASSIGN_MIN_MAX(LogicalType::TYPE_DATETIME); } return Status::NotFound("OrcMinMaxFilter: date column stats not found"); diff --git a/be/test/formats/orc/orc_chunk_reader_test.cpp b/be/test/formats/orc/orc_chunk_reader_test.cpp index 2c8679d74994bc..b66eb2b8a045ec 100644 --- a/be/test/formats/orc/orc_chunk_reader_test.cpp +++ b/be/test/formats/orc/orc_chunk_reader_test.cpp @@ -1022,19 +1022,12 @@ TEST_F(OrcChunkReaderTest, TestTimestampPreEpochSubSecond) { } } -// The ORC stripe min/max statistics decoder shares the conversion helper with the data load path. -// For a pre-1970 (negative-epoch) bound it intentionally drops the sub-second, keeping predicate -// pushdown bounds identical to before the load-path sub-second fix. Verify a pre-1970 timestamp -// stat with a sub-second decodes to its whole second. -TEST_F(OrcChunkReaderTest, OrcMinMaxDecoderPreEpochTimestampDropsSubSecond) { - ORC_UNIQUE_PTR orc_type = orc::createPrimitiveType(orc::TypeKind::TIMESTAMP); - orc::proto::ColumnStatistics stats; - auto* ts = stats.mutable_timestampstatistics(); - ts->set_minimumutc(-152539200000); // 1965-03-02 12:00:00.000 UTC (pre-1970), whole second - ts->set_maximumutc(-152539200000); - ts->set_minimumnanos(500000); // 500us sub-second that the before-epoch bound must drop - ts->set_maximumnanos(500000); - +// Decode an ORC timestamp column statistic through OrcMinMaxDecoder and return the rendered +// (min, max) bounds. Shared by the OrcMinMaxDecoder* tests below. +static std::pair decode_ts_min_max(RuntimeState* state, ObjectPool* pool, orc::TypeKind kind, + const orc::proto::ColumnStatistics& stats, + int64_t tz_offset_in_seconds) { + ORC_UNIQUE_PTR orc_type = orc::createPrimitiveType(kind); TypeDescriptor type(TYPE_DATETIME); TDescriptorTableBuilder builder; TTupleDescriptorBuilder tuple; @@ -1043,16 +1036,155 @@ TEST_F(OrcChunkReaderTest, OrcMinMaxDecoderPreEpochTimestampDropsSubSecond) { tuple.add_slot(slot_builder.build()); tuple.build(&builder); DescriptorTbl* tbl = nullptr; - ASSERT_TRUE(DescriptorTbl::create(_runtime_state.get(), &_pool, builder.desc_tbl(), &tbl, config::vector_chunk_size) - .ok()); + CHECK(DescriptorTbl::create(state, pool, builder.desc_tbl(), &tbl, config::vector_chunk_size).ok()); SlotDescriptor* slot = tbl->get_slot_descriptor(0); auto min_col = ColumnHelper::create_column(type, false); auto max_col = ColumnHelper::create_column(type, false); - ASSERT_TRUE(OrcMinMaxDecoder::decode(slot, orc_type.get(), stats, min_col.get(), max_col.get(), 0).ok()); + CHECK(OrcMinMaxDecoder::decode(slot, orc_type.get(), stats, min_col.get(), max_col.get(), tz_offset_in_seconds) + .ok()); + return {down_cast(min_col.get())->get_data()[0].to_string(), + down_cast(max_col.get())->get_data()[0].to_string()}; +} + +// A pre-1970 (negative-epoch) ORC stripe min/max TIMESTAMP statistic must decode with its +// sub-second preserved so the pruning bounds [min, max] still contain the true value range. +// Exercises the floor split of negative milliseconds, the ORC nanos +1 undo, min floor and +// max ceil to microsecond. minimumnanos/maximumnanos are stored with the writer's +1 offset. +TEST_F(OrcChunkReaderTest, OrcMinMaxDecoderPreEpochTimestampSubSecond) { + orc::proto::ColumnStatistics stats; + auto* ts = stats.mutable_timestampstatistics(); + // min: 1965-03-02 12:00:00 + 123 ms + 456789 ns sub-millisecond -> floors to .123456 + ts->set_minimumutc(-152539199877); // -152539200000 + 123 ms + ts->set_minimumnanos(456790); // 456789 ns + 1 + // max: 1965-03-02 12:00:00 + 800 ms + 654321 ns sub-millisecond -> ceils to .800655 + ts->set_maximumutc(-152539199200); // -152539200000 + 800 ms + ts->set_maximumnanos(654322); // 654321 ns + 1 + + auto [min, max] = decode_ts_min_max(_runtime_state.get(), &_pool, orc::TypeKind::TIMESTAMP, stats, 0); + EXPECT_EQ(min, "1965-03-02 12:00:00.123456"); + EXPECT_EQ(max, "1965-03-02 12:00:00.800655"); +} + +// The ORC nanos field is serialized with a +1 offset and OMITTED when equal to its default +// (0 for min, 999999 for max). Verify the +1 is undone for a present field and the conservative +// default is used for an absent field (absent max must NOT understate to .000000). +TEST_F(OrcChunkReaderTest, OrcMinMaxDecoderTimestampNanosOffsetAndDefaults) { + orc::proto::ColumnStatistics stats; + auto* ts = stats.mutable_timestampstatistics(); + ts->set_minimumutc(1621905520000); // 2021-05-25 01:18:40 + ts->set_maximumutc(1621905520000); + ts->set_minimumnanos(250001); // present: 250000 ns + 1 -> .000250 + // maximumnanos absent -> default 999999 ns -> ceils to .001000 (not .000000) + + auto [min, max] = decode_ts_min_max(_runtime_state.get(), &_pool, orc::TypeKind::TIMESTAMP, stats, 0); + EXPECT_EQ(min, "2021-05-25 01:18:40.000250"); + EXPECT_EQ(max, "2021-05-25 01:18:40.001000"); +} + +// A pre-1970 TIMESTAMP_INSTANT bound must apply the reader timezone offset on the before-epoch +// branch (it used to be dropped). The offset is folded into the seconds and decoded as plain UTC. +TEST_F(OrcChunkReaderTest, OrcMinMaxDecoderInstantTimestampOffset) { + orc::proto::ColumnStatistics stats; + auto* ts = stats.mutable_timestampstatistics(); + ts->set_minimumutc(-152539200000); // 1965-03-02 12:00:00 UTC + ts->set_maximumutc(-152539200000); + ts->set_minimumnanos(500001); // 500000 ns + 1 + ts->set_maximumnanos(500001); + + auto [min8, max8] = + decode_ts_min_max(_runtime_state.get(), &_pool, orc::TypeKind::TIMESTAMP_INSTANT, stats, 8 * 3600); + EXPECT_EQ(min8, "1965-03-02 20:00:00.000500"); // shifted +8h + EXPECT_EQ(max8, "1965-03-02 20:00:00.000500"); + + auto [min0, max0] = decode_ts_min_max(_runtime_state.get(), &_pool, orc::TypeKind::TIMESTAMP_INSTANT, stats, 0); + EXPECT_EQ(min0, "1965-03-02 12:00:00.000500"); // control: no offset + EXPECT_EQ(max0, "1965-03-02 12:00:00.000500"); +} + +// A post-1970 (after-epoch) TIMESTAMP_INSTANT with a sub-second: the offset fold lands on the +// after-epoch path while the min still floors and the max still ceils to microsecond. +TEST_F(OrcChunkReaderTest, OrcMinMaxDecoderPostEpochInstantSubSecond) { + orc::proto::ColumnStatistics stats; + auto* ts = stats.mutable_timestampstatistics(); + ts->set_minimumutc(1621905520000); // 2021-05-25 01:18:40 UTC + ts->set_maximumutc(1621905520000); + ts->set_minimumnanos(250001); // 250000 ns + 1 -> floors to .000250 + ts->set_maximumnanos(654322); // 654321 ns + 1 -> ceils to .000655 + + auto [min, max] = + decode_ts_min_max(_runtime_state.get(), &_pool, orc::TypeKind::TIMESTAMP_INSTANT, stats, 8 * 3600); + EXPECT_EQ(min, "2021-05-25 09:18:40.000250"); // shifted +8h, min floored + EXPECT_EQ(max, "2021-05-25 09:18:40.000655"); // shifted +8h, max ceiled +} - EXPECT_EQ(down_cast(min_col.get())->get_data()[0].to_string(), "1965-03-02 12:00:00"); - EXPECT_EQ(down_cast(max_col.get())->get_data()[0].to_string(), "1965-03-02 12:00:00"); +// The instant offset is folded BEFORE the epoch-sign test, so the folded seconds (not the raw +// seconds) select the before/after-epoch branch. A bound at/after the epoch can fold to the +// before-epoch path and vice versa. +TEST_F(OrcChunkReaderTest, OrcMinMaxDecoderInstantOffsetCrossesEpoch) { + { + // minimumutc = 0 (epoch) with a -8h offset folds negative -> before-epoch UTC path. + orc::proto::ColumnStatistics stats; + auto* ts = stats.mutable_timestampstatistics(); + ts->set_minimumutc(0); + ts->set_maximumutc(0); + auto [min, max] = + decode_ts_min_max(_runtime_state.get(), &_pool, orc::TypeKind::TIMESTAMP_INSTANT, stats, -8 * 3600); + EXPECT_EQ(min, "1969-12-31 16:00:00"); + } + { + // maximumutc = -1 ms (just before epoch) with a +8h offset: the default-max ceil carries + // a whole second up to the epoch, then the fold routes it to the after-epoch path. + orc::proto::ColumnStatistics stats; + auto* ts = stats.mutable_timestampstatistics(); + ts->set_minimumutc(-1); + ts->set_maximumutc(-1); + auto [min, max] = + decode_ts_min_max(_runtime_state.get(), &_pool, orc::TypeKind::TIMESTAMP_INSTANT, stats, 8 * 3600); + EXPECT_EQ(max, "1970-01-01 08:00:00"); + } +} + +// ORC is untrusted external input: a present nanos field is legitimately in [1, 1000000]. A +// malformed value outside that range must fall back to the conservative default rather than +// produce a negative/overflowing nanos. +TEST_F(OrcChunkReaderTest, OrcMinMaxDecoderTimestampMalformedNanos) { + orc::proto::ColumnStatistics stats; + auto* ts = stats.mutable_timestampstatistics(); + ts->set_minimumutc(1621905520000); // 2021-05-25 01:18:40 + ts->set_maximumutc(1621905520000); + ts->set_minimumnanos(0); // malformed (< 1) -> default 0 + ts->set_maximumnanos(2000000); // malformed (> 1000000) -> default 999999 -> ceil .001000 + + auto [min, max] = decode_ts_min_max(_runtime_state.get(), &_pool, orc::TypeKind::TIMESTAMP, stats, 0); + EXPECT_EQ(min, "2021-05-25 01:18:40"); + EXPECT_EQ(max, "2021-05-25 01:18:40.001000"); +} + +// Regression guard: the common millisecond-precision path and whole-second negatives are +// unchanged. For millisecond data the writer omits minimumnanos (sub-ms == 0) and stores +// maximumnanos == 1 (0 + 1). +TEST_F(OrcChunkReaderTest, OrcMinMaxDecoderTimestampNoRegression) { + { + orc::proto::ColumnStatistics stats; + auto* ts = stats.mutable_timestampstatistics(); + ts->set_minimumutc(1621905520500); // 2021-05-25 01:18:40.500 + ts->set_maximumutc(1621905520500); + ts->set_maximumnanos(1); // 0 + 1 + auto [min, max] = decode_ts_min_max(_runtime_state.get(), &_pool, orc::TypeKind::TIMESTAMP, stats, 0); + EXPECT_EQ(min, "2021-05-25 01:18:40.500000"); + EXPECT_EQ(max, "2021-05-25 01:18:40.500000"); + } + { + orc::proto::ColumnStatistics stats; + auto* ts = stats.mutable_timestampstatistics(); + ts->set_minimumutc(-8444232248000); // 1702-05-31 19:55:52 UTC, whole second + ts->set_maximumutc(-8444232248000); + ts->set_maximumnanos(1); // 0 + 1 + auto [min, max] = decode_ts_min_max(_runtime_state.get(), &_pool, orc::TypeKind::TIMESTAMP, stats, 0); + EXPECT_EQ(min, "1702-05-31 19:55:52"); + EXPECT_EQ(max, "1702-05-31 19:55:52"); + } } /**