From 83040e8fc4f105e05c0629b43487dffbd1df72d8 Mon Sep 17 00:00:00 2001 From: "lisizhuo.lsz" Date: Fri, 19 Dec 2025 09:18:49 +0000 Subject: [PATCH 1/2] feat(scan): support built-in global index search during scan process --- include/paimon/defs.h | 2 + .../global_index/bitmap_global_index_result.h | 6 +- .../paimon/global_index/global_index_result.h | 4 + .../paimon/global_index/global_index_scan.h | 12 +- include/paimon/scan_context.h | 27 +- include/paimon/utils/range.h | 9 + src/paimon/CMakeLists.txt | 1 + src/paimon/common/defs.cpp | 2 +- .../bitmap_global_index_result.cpp | 9 +- .../bitmap_global_index_result_test.cpp | 10 +- .../global_index/global_index_result.cpp | 25 + .../global_index/global_index_result_test.cpp | 7 + src/paimon/common/utils/range.cpp | 37 + src/paimon/common/utils/range_test.cpp | 125 +++ src/paimon/core/core_options.cpp | 7 + src/paimon/core/core_options.h | 1 + src/paimon/core/core_options_test.cpp | 3 + .../core/global_index/global_index_scan.cpp | 99 ++- .../global_index/global_index_scan_impl.cpp | 127 ++- .../global_index/global_index_scan_impl.h | 23 +- .../core/global_index/indexed_split_impl.h | 5 +- .../core/global_index/indexed_split_test.cpp | 24 +- src/paimon/core/io/data_file_meta.cpp | 6 +- src/paimon/core/io/data_file_meta.h | 2 +- src/paimon/core/io/data_file_meta_test.cpp | 9 +- .../operation/abstract_file_store_write.cpp | 3 +- .../core/operation/abstract_split_read.cpp | 4 +- .../core/operation/abstract_split_read.h | 6 +- .../data_evolution_file_store_scan.cpp | 16 +- .../data_evolution_file_store_scan.h | 4 +- .../data_evolution_file_store_scan_test.cpp | 35 +- .../operation/data_evolution_split_read.cpp | 13 +- .../operation/data_evolution_split_read.h | 7 +- .../core/operation/file_store_commit_impl.cpp | 6 +- src/paimon/core/operation/file_store_scan.cpp | 1 - src/paimon/core/operation/file_store_scan.h | 31 +- .../core/operation/file_store_write.cpp | 4 +- .../key_value_file_store_scan_test.cpp | 42 +- .../core/operation/merge_file_split_read.cpp | 7 +- .../core/operation/merge_file_split_read.h | 2 +- .../core/operation/raw_file_split_read.cpp | 2 +- .../core/operation/raw_file_split_read.h | 2 +- src/paimon/core/operation/scan_context.cpp | 15 +- .../core/operation/scan_context_test.cpp | 8 +- src/paimon/core/schema/table_schema.cpp | 14 +- src/paimon/core/schema/table_schema.h | 2 +- src/paimon/core/schema/table_schema_test.cpp | 4 +- .../source/data_evolution_batch_scan.cpp | 150 ++++ .../table/source/data_evolution_batch_scan.h | 54 ++ .../core/table/source/data_table_batch_scan.h | 12 + .../table/source/snapshot/snapshot_reader.h | 13 + src/paimon/core/table/source/table_scan.cpp | 17 +- .../global_index/lumina/lumina_global_index.h | 26 +- src/paimon/testing/utils/data_generator.cpp | 15 +- src/paimon/testing/utils/data_generator.h | 2 - test/inte/blob_table_inte_test.cpp | 40 +- test/inte/data_evolution_table_test.cpp | 92 +- test/inte/global_index_test.cpp | 787 +++++++++++++++--- 58 files changed, 1551 insertions(+), 467 deletions(-) create mode 100644 src/paimon/core/table/source/data_evolution_batch_scan.cpp create mode 100644 src/paimon/core/table/source/data_evolution_batch_scan.h diff --git a/include/paimon/defs.h b/include/paimon/defs.h index 8cfece45f..bab1f5825 100644 --- a/include/paimon/defs.h +++ b/include/paimon/defs.h @@ -282,6 +282,8 @@ struct PAIMON_EXPORT Options { /// "blob-as-descriptor" - Read and write blob field using blob descriptor rather than blob /// bytes. Default value is "false". static const char BLOB_AS_DESCRIPTOR[]; + /// "global-index.enabled" - Whether to enable global index for scan. Default value is "true". + static const char GLOBAL_INDEX_ENABLED[]; }; static constexpr int64_t BATCH_WRITE_COMMIT_IDENTIFIER = std::numeric_limits::max(); diff --git a/include/paimon/global_index/bitmap_global_index_result.h b/include/paimon/global_index/bitmap_global_index_result.h index 9ef551b17..192fde662 100644 --- a/include/paimon/global_index/bitmap_global_index_result.h +++ b/include/paimon/global_index/bitmap_global_index_result.h @@ -78,9 +78,9 @@ class PAIMON_EXPORT BitmapGlobalIndexResult : public GlobalIndexResult { /// bitmap is not actually required. **Not thread-safe**. Result GetBitmap() const; - /// @return A shared pointer to a new BitmapGlobalIndexResult instance representing the given - /// inclusive range [from, to]. - static std::shared_ptr FromRange(const Range& range); + /// Creates `BitmapGlobalIndexResult` for all row ids in the given ranges. + /// @note Overlapping or unsorted ranges are accepted. + static std::shared_ptr FromRanges(const std::vector& ranges); private: mutable bool initialized_ = false; diff --git a/include/paimon/global_index/global_index_result.h b/include/paimon/global_index/global_index_result.h index 78f71c8fc..75b7cf48b 100644 --- a/include/paimon/global_index/global_index_result.h +++ b/include/paimon/global_index/global_index_result.h @@ -23,6 +23,7 @@ #include "paimon/memory/bytes.h" #include "paimon/memory/memory_pool.h" #include "paimon/result.h" +#include "paimon/utils/range.h" #include "paimon/visibility.h" namespace paimon { @@ -55,6 +56,9 @@ class PAIMON_EXPORT GlobalIndexResult : public std::enable_shared_from_this> CreateIterator() const = 0; + /// Returns non-overlapping, sorted ranges covering all row ids in `GlobalIndexResult`. + Result> ToRanges() const; + /// Computes the logical AND (intersection) between current result and another. virtual Result> And( const std::shared_ptr& other); diff --git a/include/paimon/global_index/global_index_scan.h b/include/paimon/global_index/global_index_scan.h index 35d340fcb..d561aa696 100644 --- a/include/paimon/global_index/global_index_scan.h +++ b/include/paimon/global_index/global_index_scan.h @@ -37,7 +37,7 @@ class PAIMON_EXPORT GlobalIndexScan { /// /// @param table_path Root directory of the table. /// @param snapshot_id Optional snapshot ID to read from; if not provided, uses the latest. - /// @param partitions Optional list of partition specs to restrict the scan scope. + /// @param partitions Optional list of specific partitions to restrict the scan scope. /// Each map represents one partition (e.g., {"dt": "2024-06-01"}). /// If omitted, scans all partitions. /// @param options Index-specific configuration. @@ -53,6 +53,16 @@ class PAIMON_EXPORT GlobalIndexScan { const std::map& options, const std::shared_ptr& file_system, const std::shared_ptr& pool); + /// Creates a `GlobalIndexScan` instance for the specified table and context. + /// + /// @param partition_filters Optional specific partition predicates. + static Result> Create( + const std::string& root_path, const std::optional& snapshot_id, + const std::shared_ptr& partition_filters, + const std::map& options, + const std::shared_ptr& file_system, + const std::shared_ptr& memory_pool); + virtual ~GlobalIndexScan() = default; /// Creates a scanner for the global index over the specified row ID range. diff --git a/include/paimon/scan_context.h b/include/paimon/scan_context.h index c26131840..686d5e3da 100644 --- a/include/paimon/scan_context.h +++ b/include/paimon/scan_context.h @@ -23,12 +23,11 @@ #include #include +#include "paimon/global_index/global_index_result.h" #include "paimon/predicate/predicate.h" #include "paimon/result.h" #include "paimon/type_fwd.h" -#include "paimon/utils/range.h" #include "paimon/visibility.h" - namespace paimon { class ScanContextBuilder; class ScanFilter; @@ -45,6 +44,7 @@ class PAIMON_EXPORT ScanContext { public: ScanContext(const std::string& path, bool is_streaming_mode, std::optional limit, const std::shared_ptr& scan_filter, + const std::shared_ptr& global_index_result, const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::map& options); @@ -77,12 +77,16 @@ class PAIMON_EXPORT ScanContext { std::shared_ptr GetExecutor() const { return executor_; } + std::shared_ptr GetGlobalIndexResult() const { + return global_index_result_; + } private: std::string path_; bool is_streaming_mode_; std::optional limit_; std::shared_ptr scan_filters_; + std::shared_ptr global_index_result_; std::shared_ptr memory_pool_; std::shared_ptr executor_; std::map options_; @@ -93,11 +97,10 @@ class PAIMON_EXPORT ScanFilter { public: ScanFilter(const std::shared_ptr& predicate, const std::vector>& partition_filters, - const std::optional& bucket_filter, const std::vector& row_ranges) + const std::optional& bucket_filter) : predicates_(predicate), bucket_filter_(bucket_filter), - partition_filters_(partition_filters), - row_ranges_(row_ranges) {} + partition_filters_(partition_filters) {} std::shared_ptr GetPredicate() const { return predicates_; @@ -109,15 +112,10 @@ class PAIMON_EXPORT ScanFilter { return partition_filters_; } - const std::vector& GetRowRanges() const { - return row_ranges_; - } - private: std::shared_ptr predicates_; std::optional bucket_filter_; std::vector> partition_filters_; - std::vector row_ranges_; }; /// `ScanContextBuilder` used to build a `ScanContext`, has input validation. @@ -138,10 +136,11 @@ class PAIMON_EXPORT ScanContextBuilder { /// Set a predicate for filtering data. ScanContextBuilder& SetPredicate(const std::shared_ptr& predicate); - /// Specify the row id ranges for scan. This is usually used to read specific rows in - /// data-evolution mode. File ranges that do not have any intersection with range_ids will be - /// filtered. If not set, all rows are returned - ScanContextBuilder& SetRowRanges(const std::vector& row_ranges); + /// Sets the result of a global index search (e.g., row ids (may with scores) from a distributed + /// index lookup). This is used to push down index-filtered row ids into the scan for efficient + /// data retrieval. + ScanContextBuilder& SetGlobalIndexResult( + const std::shared_ptr& global_index_result); /// The options added or set in `ScanContextBuilder` have high priority and will be merged with /// the options in table schema. ScanContextBuilder& AddOption(const std::string& key, const std::string& value); diff --git a/include/paimon/utils/range.h b/include/paimon/utils/range.h index 9cf1dd70f..90b7b98a2 100644 --- a/include/paimon/utils/range.h +++ b/include/paimon/utils/range.h @@ -46,6 +46,15 @@ struct PAIMON_EXPORT Range { /// Computes the set intersection of two collections of disjoint, sorted ranges. static std::vector And(const std::vector& left, const std::vector& right); + /// Excludes the given ranges from this range and returns the remaining ranges. + /// + /// For example, if this range is [0, 10000] and ranges to exclude are [1000, 2000], [3000, + /// 4000], [5000, 6000], then the result is [0, 999], [2001, 2999], [4001, 4999], [6001, 10000]. + /// + /// @param ranges The ranges to exclude (can be unsorted and overlapping). + /// @return The remaining ranges after exclusion. + std::vector Exclude(const std::vector& ranges) const; + bool operator==(const Range& other) const; bool operator<(const Range& other) const; diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 96010e981..0b75fcb3e 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -241,6 +241,7 @@ set(PAIMON_CORE_SRCS core/table/source/startup_mode.cpp core/table/source/table_read.cpp core/table/source/table_scan.cpp + core/table/source/data_evolution_batch_scan.cpp core/utils/field_mapping.cpp core/utils/fields_comparator.cpp core/utils/file_store_path_factory.cpp diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp index 632462a66..c198531a1 100644 --- a/src/paimon/common/defs.cpp +++ b/src/paimon/common/defs.cpp @@ -79,5 +79,5 @@ const char Options::ROW_TRACKING_ENABLED[] = "row-tracking.enabled"; const char Options::DATA_EVOLUTION_ENABLED[] = "data-evolution.enabled"; const char Options::PARTITION_GENERATE_LEGACY_NAME[] = "partition.legacy-name"; const char Options::BLOB_AS_DESCRIPTOR[] = "blob-as-descriptor"; - +const char Options::GLOBAL_INDEX_ENABLED[] = "global-index.enabled"; } // namespace paimon diff --git a/src/paimon/common/global_index/bitmap_global_index_result.cpp b/src/paimon/common/global_index/bitmap_global_index_result.cpp index dd7a7ad41..74a4d1e94 100644 --- a/src/paimon/common/global_index/bitmap_global_index_result.cpp +++ b/src/paimon/common/global_index/bitmap_global_index_result.cpp @@ -74,10 +74,13 @@ std::string BitmapGlobalIndexResult::ToString() const { return bitmap.value()->ToString(); } -std::shared_ptr BitmapGlobalIndexResult::FromRange(const Range& range) { - BitmapGlobalIndexResult::BitmapSupplier supplier = [range]() -> Result { +std::shared_ptr BitmapGlobalIndexResult::FromRanges( + const std::vector& ranges) { + BitmapGlobalIndexResult::BitmapSupplier supplier = [ranges]() -> Result { RoaringBitmap64 bitmap; - bitmap.AddRange(range.from, range.to + 1); + for (const auto& range : ranges) { + bitmap.AddRange(range.from, range.to + 1); + } return bitmap; }; return std::make_shared(supplier); diff --git a/src/paimon/common/global_index/bitmap_global_index_result_test.cpp b/src/paimon/common/global_index/bitmap_global_index_result_test.cpp index 847d32aea..237f39850 100644 --- a/src/paimon/common/global_index/bitmap_global_index_result_test.cpp +++ b/src/paimon/common/global_index/bitmap_global_index_result_test.cpp @@ -164,14 +164,18 @@ TEST_F(BitmapGlobalIndexResultTest, TestInvalidBitmapResult) { ASSERT_TRUE(result->ToString().find("Invalid: invalid supplier") != std::string::npos); } -TEST_F(BitmapGlobalIndexResultTest, TestFromRange) { +TEST_F(BitmapGlobalIndexResultTest, TestFromRanges) { { - auto result = BitmapGlobalIndexResult::FromRange(Range(0, 5)); + auto result = BitmapGlobalIndexResult::FromRanges({Range(0, 5)}); ASSERT_EQ(result->ToString(), "{0,1,2,3,4,5}"); } { - auto result = BitmapGlobalIndexResult::FromRange(Range(10, 10)); + auto result = BitmapGlobalIndexResult::FromRanges({Range(10, 10)}); ASSERT_EQ(result->ToString(), "{10}"); } + { + auto result = BitmapGlobalIndexResult::FromRanges({Range(0, 5), Range(10, 10)}); + ASSERT_EQ(result->ToString(), "{0,1,2,3,4,5,10}"); + } } } // namespace paimon::test diff --git a/src/paimon/common/global_index/global_index_result.cpp b/src/paimon/common/global_index/global_index_result.cpp index 4d89bf730..b9141f7e7 100644 --- a/src/paimon/common/global_index/global_index_result.cpp +++ b/src/paimon/common/global_index/global_index_result.cpp @@ -133,4 +133,29 @@ Result> GlobalIndexResult::Deserialize( return std::make_shared(std::move(bitmap), std::move(scores)); } +Result> GlobalIndexResult::ToRanges() const { + std::vector ranges; + PAIMON_ASSIGN_OR_RAISE(bool empty, IsEmpty()); + if (empty) { + return ranges; + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr iter, CreateIterator()); + int64_t range_start = iter->Next(); + int64_t range_end = range_start; + while (iter->HasNext()) { + int64_t current = iter->Next(); + if (current == range_end + 1) { + // Extend the current range + range_end = current; + } else { + ranges.push_back(Range(range_start, range_end)); + range_start = current; + range_end = current; + } + } + // Add the last range + ranges.push_back(Range(range_start, range_end)); + return ranges; +} + } // namespace paimon diff --git a/src/paimon/common/global_index/global_index_result_test.cpp b/src/paimon/common/global_index/global_index_result_test.cpp index c90cbe7a4..546a8080c 100644 --- a/src/paimon/common/global_index/global_index_result_test.cpp +++ b/src/paimon/common/global_index/global_index_result_test.cpp @@ -73,9 +73,16 @@ TEST_F(GlobalIndexResultTest, TestSimple) { std::make_shared(std::vector({100, 5, 4, 3, 200})); ASSERT_OK_AND_ASSIGN(auto and_result, result1->And(result2)); ASSERT_EQ(and_result->ToString(), "{3,5,100}"); + ASSERT_OK_AND_ASSIGN(auto and_ranges, and_result->ToRanges()); + std::vector expect_and_ranges = {Range(3, 3), Range(5, 5), Range(100, 100)}; + ASSERT_EQ(and_ranges, expect_and_ranges); ASSERT_OK_AND_ASSIGN(auto or_result, result1->Or(result2)); ASSERT_EQ(or_result->ToString(), "{1,3,4,5,100,200}"); + ASSERT_OK_AND_ASSIGN(auto or_ranges, or_result->ToRanges()); + std::vector expect_or_ranges = {Range(1, 1), Range(3, 5), Range(100, 100), + Range(200, 200)}; + ASSERT_EQ(or_ranges, expect_or_ranges); } TEST_F(GlobalIndexResultTest, TestSerializeAndDeserializeSimple) { diff --git a/src/paimon/common/utils/range.cpp b/src/paimon/common/utils/range.cpp index abecbce6f..793dfa9b0 100644 --- a/src/paimon/common/utils/range.cpp +++ b/src/paimon/common/utils/range.cpp @@ -101,6 +101,43 @@ bool Range::HasIntersection(const Range& left, const Range& right) { return intersection_start <= intersection_end; } +std::vector Range::Exclude(const std::vector& ranges) const { + if (ranges.empty()) { + return {*this}; + } + + // Sort ranges by from + std::vector sorted = ranges; + std::sort(sorted.begin(), sorted.end(), + [](const Range& left, const Range& right) { return left.from < right.from; }); + + std::vector result; + int64_t current = from; + + for (const auto& exclude : sorted) { + // Compute intersection with the current range + auto intersect = Range::Intersection(Range(current, to), exclude); + if (!intersect) { + continue; + } + // Add the part before the intersection (if any) + if (current < intersect.value().from) { + result.push_back(Range(current, intersect.value().from - 1)); + } + // Move current position past the intersection + current = intersect.value().to + 1; + if (current > to) { + break; + } + } + // Add the remaining part after all exclusions (if any) + if (current <= to) { + result.push_back(Range(current, to)); + } + + return result; +} + bool Range::operator==(const Range& other) const { if (this == &other) { return true; diff --git a/src/paimon/common/utils/range_test.cpp b/src/paimon/common/utils/range_test.cpp index 38fb2bb44..ac34d35f5 100644 --- a/src/paimon/common/utils/range_test.cpp +++ b/src/paimon/common/utils/range_test.cpp @@ -253,4 +253,129 @@ TEST(RangeTest, TestAnd) { ASSERT_EQ(result, expected); } } + +TEST(RangeTest, TestExclude) { + { + // test basic + // [0, 10000] exclude [1000,2000],[3000,4000],[5000,6000] + // Expected: [0, 999],[2001,2999],[4001,4999],[6001, 10000] + Range range(0, 10000); + std::vector excludes = {Range(1000, 2000), Range(3000, 4000), Range(5000, 6000)}; + auto result = range.Exclude(excludes); + std::vector expected = {Range(0, 999), Range(2001, 2999), Range(4001, 4999), + Range(6001, 10000)}; + ASSERT_EQ(result, expected); + } + { + // Same as basic but with unsorted exclusions + Range range(0, 10000); + std::vector excludes = {Range(5000, 6000), Range(1000, 2000), Range(3000, 4000)}; + auto result = range.Exclude(excludes); + std::vector expected = {Range(0, 999), Range(2001, 2999), Range(4001, 4999), + Range(6001, 10000)}; + ASSERT_EQ(result, expected); + } + { + // exclude empty exclusions + Range range(100, 200); + std::vector excludes = {}; + auto result = range.Exclude(excludes); + std::vector expected = {Range(100, 200)}; + ASSERT_EQ(result, expected); + } + { + // exclude no intersection + Range range(100, 200); + std::vector excludes = {Range(300, 400), Range(500, 600)}; + auto result = range.Exclude(excludes); + std::vector expected = {Range(100, 200)}; + ASSERT_EQ(result, expected); + } + { + // exclude at start + Range range(0, 100); + std::vector excludes = {Range(0, 10)}; + auto result = range.Exclude(excludes); + std::vector expected = {Range(11, 100)}; + ASSERT_EQ(result, expected); + } + { + // exclude at end + Range range(0, 100); + std::vector excludes = {Range(90, 100)}; + auto result = range.Exclude(excludes); + std::vector expected = {Range(0, 89)}; + ASSERT_EQ(result, expected); + } + { + // exclusion extends past the end of the range + Range range(100, 200); + std::vector excludes = {Range(150, 300)}; + auto result = range.Exclude(excludes); + std::vector expected = {Range(100, 149)}; + ASSERT_EQ(result, expected); + } + { + // exclusion starts before the range + Range range(100, 200); + std::vector excludes = {Range(50, 150)}; + auto result = range.Exclude(excludes); + std::vector expected = {Range(151, 200)}; + ASSERT_EQ(result, expected); + } + { + // overlapping exclusion ranges + Range range(0, 100); + std::vector excludes = {Range(20, 50), Range(40, 70)}; + auto result = range.Exclude(excludes); + std::vector expected = {Range(0, 19), Range(71, 100)}; + ASSERT_EQ(result, expected); + } + { + // exclusion completely covers the range + Range range(50, 60); + std::vector excludes = {Range(0, 100)}; + auto result = range.Exclude(excludes); + ASSERT_TRUE(result.empty()); + } + { + // exclusion completely matches the range + Range range(50, 60); + std::vector excludes = {Range(50, 60)}; + auto result = range.Exclude(excludes); + ASSERT_TRUE(result.empty()); + } + { + // single exclusion in the middle + Range range(0, 100); + std::vector excludes = {Range(40, 60)}; + auto result = range.Exclude(excludes); + std::vector expected = {Range(0, 39), Range(61, 100)}; + ASSERT_EQ(result, expected); + } + { + // adjacent exclusion ranges + Range range(0, 100); + std::vector excludes = {Range(20, 30), Range(31, 40)}; + auto result = range.Exclude(excludes); + std::vector expected = {Range(0, 19), Range(41, 100)}; + ASSERT_EQ(result, expected); + } + { + // range is a single point + Range range(50, 50); + std::vector excludes = {Range(50, 50)}; + auto result = range.Exclude(excludes); + ASSERT_TRUE(result.empty()); + } + { + // single point exclusion + Range range(0, 100); + std::vector excludes = {Range(50, 50)}; + auto result = range.Exclude(excludes); + std::vector expected = {Range(0, 49), Range(51, 100)}; + ASSERT_EQ(result, expected); + } +} + } // namespace paimon::test diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp index 8e6788212..c28cc1aaf 100644 --- a/src/paimon/core/core_options.cpp +++ b/src/paimon/core/core_options.cpp @@ -302,6 +302,7 @@ struct CoreOptions::Impl { bool row_tracking_enabled = false; bool data_evolution_enabled = false; bool legacy_partition_name_enabled = true; + bool global_index_enabled = true; }; // Parse configurations from a map and return a populated CoreOptions object @@ -466,6 +467,9 @@ Result CoreOptions::FromMap( // Parse partition.legacy-name PAIMON_RETURN_NOT_OK(parser.Parse(Options::PARTITION_GENERATE_LEGACY_NAME, &impl->legacy_partition_name_enabled)); + // Parse global-index.enabled + PAIMON_RETURN_NOT_OK( + parser.Parse(Options::GLOBAL_INDEX_ENABLED, &impl->global_index_enabled)); return options; } @@ -739,4 +743,7 @@ bool CoreOptions::LegacyPartitionNameEnabled() const { return impl_->legacy_partition_name_enabled; } +bool CoreOptions::GlobalIndexEnabled() const { + return impl_->global_index_enabled; +} } // namespace paimon diff --git a/src/paimon/core/core_options.h b/src/paimon/core/core_options.h index 1efd60571..44608b629 100644 --- a/src/paimon/core/core_options.h +++ b/src/paimon/core/core_options.h @@ -116,6 +116,7 @@ class PAIMON_EXPORT CoreOptions { bool LegacyPartitionNameEnabled() const; + bool GlobalIndexEnabled() const; const std::map& ToMap() const; private: diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp index d6ca29434..09ef160f2 100644 --- a/src/paimon/core/core_options_test.cpp +++ b/src/paimon/core/core_options_test.cpp @@ -86,6 +86,7 @@ TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_FALSE(core_options.RowTrackingEnabled()); ASSERT_FALSE(core_options.DataEvolutionEnabled()); ASSERT_TRUE(core_options.LegacyPartitionNameEnabled()); + ASSERT_TRUE(core_options.GlobalIndexEnabled()); } TEST(CoreOptionsTest, TestFromMap) { @@ -142,6 +143,7 @@ TEST(CoreOptionsTest, TestFromMap) { {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::PARTITION_GENERATE_LEGACY_NAME, "false"}, + {Options::GLOBAL_INDEX_ENABLED, "false"}, }; ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); @@ -209,6 +211,7 @@ TEST(CoreOptionsTest, TestFromMap) { ASSERT_TRUE(core_options.RowTrackingEnabled()); ASSERT_TRUE(core_options.DataEvolutionEnabled()); ASSERT_FALSE(core_options.LegacyPartitionNameEnabled()); + ASSERT_FALSE(core_options.GlobalIndexEnabled()); } TEST(CoreOptionsTest, TestInvalidCase) { diff --git a/src/paimon/core/global_index/global_index_scan.cpp b/src/paimon/core/global_index/global_index_scan.cpp index 1e4b76990..133fa3e12 100644 --- a/src/paimon/core/global_index/global_index_scan.cpp +++ b/src/paimon/core/global_index/global_index_scan.cpp @@ -18,9 +18,53 @@ #include "paimon/core/core_options.h" #include "paimon/core/global_index/global_index_scan_impl.h" +#include "paimon/core/operation/file_store_scan.h" #include "paimon/core/schema/schema_manager.h" - +#include "paimon/core/utils/snapshot_manager.h" namespace paimon { +namespace { +Result> LoadSchema(const std::string& root_path, + const std::map& options, + const std::shared_ptr& file_system) { + PAIMON_ASSIGN_OR_RAISE( + CoreOptions tmp_options, + CoreOptions::FromMap(options, /*fs_scheme_to_identifier_map=*/{}, file_system)); + SchemaManager schema_manager(tmp_options.GetFileSystem(), root_path); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest_table_schema, + schema_manager.Latest()); + if (latest_table_schema == std::nullopt) { + return Status::Invalid("not found latest schema"); + } + return latest_table_schema.value(); +} + +Result MergeOptions(const std::shared_ptr& table_schema, + const std::map& options, + const std::shared_ptr& file_system) { + auto final_options = table_schema->Options(); + for (const auto& [key, value] : options) { + final_options[key] = value; + } + return CoreOptions::FromMap(final_options, /*fs_scheme_to_identifier_map=*/{}, file_system); +} + +Result LoadSnapshot(const std::string& root_path, + const std::optional& snapshot_id, + const CoreOptions& core_options) { + std::optional snapshot; + SnapshotManager snapshot_manager(core_options.GetFileSystem(), root_path); + if (snapshot_id) { + PAIMON_ASSIGN_OR_RAISE(snapshot, snapshot_manager.LoadSnapshot(snapshot_id.value())); + } else { + PAIMON_ASSIGN_OR_RAISE(snapshot, snapshot_manager.LatestSnapshot()); + } + if (!snapshot) { + return Status::Invalid("not found latest snapshot"); + } + return snapshot.value(); +} +} // namespace + Result> GlobalIndexScan::Create( const std::string& root_path, const std::optional& snapshot_id, const std::optional>>& partitions, @@ -32,26 +76,43 @@ Result> GlobalIndexScan::Create( "invalid input partition, supposed to be null or at least one partition"); } std::shared_ptr pool = memory_pool ? memory_pool : GetDefaultPool(); - // load schema - PAIMON_ASSIGN_OR_RAISE( - CoreOptions tmp_options, - CoreOptions::FromMap(options, /*fs_scheme_to_identifier_map=*/{}, file_system)); - SchemaManager schema_manager(tmp_options.GetFileSystem(), root_path); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest_table_schema, - schema_manager.Latest()); - if (latest_table_schema == std::nullopt) { - return Status::Invalid("not found latest schema"); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_schema, + LoadSchema(root_path, options, file_system)); + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, + MergeOptions(table_schema, options, file_system)); + std::shared_ptr partition_filters; + if (partitions) { + auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); + PAIMON_ASSIGN_OR_RAISE(partition_filters, FileStoreScan::CreatePartitionPredicate( + table_schema->PartitionKeys(), + core_options.GetPartitionDefaultName(), + arrow_schema, partitions.value())); } - // merge options - auto final_options = latest_table_schema.value()->Options(); - for (const auto& [key, value] : options) { - final_options[key] = value; + PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, LoadSnapshot(root_path, snapshot_id, core_options)); + return std::make_unique(root_path, table_schema, snapshot, + partition_filters, core_options, pool); +} + +Result> GlobalIndexScan::Create( + const std::string& root_path, const std::optional& snapshot_id, + const std::shared_ptr& partitions, const std::map& options, + const std::shared_ptr& file_system, + const std::shared_ptr& memory_pool) { + std::shared_ptr partition_filters; + if (partitions) { + partition_filters = std::dynamic_pointer_cast(partitions); + if (!partition_filters) { + return Status::Invalid("partition filters cannot cast to PredicateFilter"); + } } - PAIMON_ASSIGN_OR_RAISE( - CoreOptions core_options, - CoreOptions::FromMap(final_options, /*fs_scheme_to_identifier_map=*/{}, file_system)); - return std::make_unique(root_path, latest_table_schema.value(), - snapshot_id, partitions, core_options, pool); + std::shared_ptr pool = memory_pool ? memory_pool : GetDefaultPool(); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_schema, + LoadSchema(root_path, options, file_system)); + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, + MergeOptions(table_schema, options, file_system)); + PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, LoadSnapshot(root_path, snapshot_id, core_options)); + return std::make_unique(root_path, table_schema, snapshot, + partition_filters, core_options, pool); } } // namespace paimon diff --git a/src/paimon/core/global_index/global_index_scan_impl.cpp b/src/paimon/core/global_index/global_index_scan_impl.cpp index e143d882a..9a90ed843 100644 --- a/src/paimon/core/global_index/global_index_scan_impl.cpp +++ b/src/paimon/core/global_index/global_index_scan_impl.cpp @@ -13,31 +13,29 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - #include "paimon/core/global_index/global_index_scan_impl.h" #include #include #include +#include "paimon/common/executor/future.h" #include "paimon/core/global_index/row_range_global_index_scanner_impl.h" #include "paimon/core/index/index_file_handler.h" -#include "paimon/core/utils/snapshot_manager.h" - +#include "paimon/global_index/bitmap_global_index_result.h" namespace paimon { -GlobalIndexScanImpl::GlobalIndexScanImpl( - const std::string& root_path, const std::shared_ptr& table_schema, - const std::optional& snapshot_id, - const std::optional>>& partitions, - const CoreOptions& options, const std::shared_ptr& pool) +GlobalIndexScanImpl::GlobalIndexScanImpl(const std::string& root_path, + const std::shared_ptr& table_schema, + const Snapshot& snapshot, + const std::shared_ptr& partitions, + const CoreOptions& options, + const std::shared_ptr& pool) : pool_(pool), root_path_(root_path), table_schema_(table_schema), - snapshot_id_(snapshot_id), + snapshot_(snapshot), partitions_(partitions), - options_(options) { - assert(!partitions || !partitions.value().empty()); -} + options_(options) {} Result> GlobalIndexScanImpl::CreateRangeScan( const Range& range) { @@ -65,22 +63,26 @@ Result> GlobalIndexScanImpl::CreateR return std::make_shared(table_schema_, index_file_path_factory, filtered_entries, options_, pool_); } + Result> GlobalIndexScanImpl::GetRowRangeList() { PAIMON_RETURN_NOT_OK(Scan()); - std::map> index_ranges; + std::map> index_type_to_ranges; + std::vector index_ranges; + index_ranges.reserve(entries_.size()); for (const auto& entry : entries_) { const auto& global_index_meta = entry.index_file->GetGlobalIndexMeta(); assert(global_index_meta); const auto& index_meta = global_index_meta.value(); - index_ranges[entry.index_file->IndexType()].emplace_back(index_meta.row_range_start, - index_meta.row_range_end); + Range range(index_meta.row_range_start, index_meta.row_range_end); + index_ranges.push_back(range); + index_type_to_ranges[entry.index_file->IndexType()].push_back(range); } std::string check_index_type; std::vector check_ranges; // check all type index have same shard ranges // If index a has [1,10],[20,30] and index b has [1,10],[20,25], it's inconsistent, because // it is hard to handle the [26,30] range. - for (const auto& [type, ranges] : index_ranges) { + for (const auto& [type, ranges] : index_type_to_ranges) { if (check_index_type.empty()) { check_index_type = type; check_ranges = Range::SortAndMergeOverlap(ranges, /*adjacent=*/true); @@ -93,8 +95,7 @@ Result> GlobalIndexScanImpl::GetRowRangeList() { } } } - - return check_ranges; + return Range::SortAndMergeOverlap(index_ranges, /*adjacent=*/false); } Status GlobalIndexScanImpl::Scan() { @@ -118,40 +119,80 @@ Status GlobalIndexScanImpl::Scan() { auto index_file_handler = std::make_unique( std::move(index_manifest_file), std::make_shared(path_factory_)); - // prepare snapshot - std::optional snapshot; - SnapshotManager snapshot_manager(options_.GetFileSystem(), root_path_); - if (snapshot_id_) { - PAIMON_ASSIGN_OR_RAISE(snapshot, snapshot_manager.LoadSnapshot(snapshot_id_.value())); - } else { - PAIMON_ASSIGN_OR_RAISE(snapshot, snapshot_manager.LatestSnapshot()); - } - if (!snapshot) { - return Status::Invalid("not found latest snapshot"); - } - - // prepare filter - std::unordered_set partitions_set; - if (partitions_) { - for (const auto& partition : partitions_.value()) { - PAIMON_ASSIGN_OR_RAISE(BinaryRow partition_row, path_factory_->ToBinaryRow(partition)); - partitions_set.insert(partition_row); - } - } + PAIMON_ASSIGN_OR_RAISE(std::vector partition_fields, + table_schema_->GetFields(table_schema_->PartitionKeys())); + auto partition_schema = DataField::ConvertDataFieldsToArrowSchema(partition_fields); std::function(const IndexManifestEntry&)> filter = - [&](const IndexManifestEntry& entry) -> bool { - if (!partitions_set.empty() && - partitions_set.find(entry.partition) == partitions_set.end()) { - return false; + [&](const IndexManifestEntry& entry) -> Result { + if (partitions_) { + PAIMON_ASSIGN_OR_RAISE(bool saved, + partitions_->Test(partition_schema, entry.partition)); + if (!saved) { + return false; + } } if (!entry.index_file->GetGlobalIndexMeta()) { return false; } return true; }; - PAIMON_ASSIGN_OR_RAISE(entries_, index_file_handler->Scan(snapshot.value(), filter)); + PAIMON_ASSIGN_OR_RAISE(entries_, index_file_handler->Scan(snapshot_, filter)); initialized_ = true; return Status::OK(); } +Result>> GlobalIndexScanImpl::ParallelScan( + const std::vector& ranges, const std::shared_ptr& predicate, + const std::shared_ptr& executor) { + std::vector> range_scanners; + range_scanners.reserve(ranges.size()); + for (const auto& range : ranges) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr scanner, + CreateRangeScan(range)); + range_scanners.push_back(scanner); + } + + std::vector>>>> futures; + for (const auto& scanner : range_scanners) { + auto search_index = + [&scanner, &predicate]() -> Result>> { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr evaluator, + scanner->CreateIndexEvaluator()); + return evaluator->Evaluate(predicate); + }; + futures.push_back(Via(executor.get(), search_index)); + } + auto collected_results = CollectAll(futures); + + // collect inner result and check all null + bool all_null = true; + std::vector>> results; + for (auto& result : collected_results) { + PAIMON_ASSIGN_OR_RAISE(std::optional> inner_result, + result); + if (inner_result) { + all_null = false; + } + results.push_back(std::move(inner_result)); + } + if (all_null) { + return std::optional>(); + } + + // union result from multiple ranges + std::shared_ptr final_global_index_result = + BitmapGlobalIndexResult::FromRanges({}); + + for (size_t i = 0; i < results.size(); ++i) { + if (results[i]) { + PAIMON_ASSIGN_OR_RAISE(final_global_index_result, + final_global_index_result->Or(results[i].value())); + } else { + PAIMON_ASSIGN_OR_RAISE( + final_global_index_result, + final_global_index_result->Or(BitmapGlobalIndexResult::FromRanges({ranges[i]}))); + } + } + return std::optional>(final_global_index_result); +} } // namespace paimon diff --git a/src/paimon/core/global_index/global_index_scan_impl.h b/src/paimon/core/global_index/global_index_scan_impl.h index 6c3077c99..602f6a66f 100644 --- a/src/paimon/core/global_index/global_index_scan_impl.h +++ b/src/paimon/core/global_index/global_index_scan_impl.h @@ -21,9 +21,11 @@ #include #include +#include "paimon/common/predicate/predicate_filter.h" #include "paimon/core/core_options.h" #include "paimon/core/manifest/index_manifest_entry.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/global_index/global_index_scan.h" @@ -31,27 +33,34 @@ namespace paimon { class GlobalIndexScanImpl : public GlobalIndexScan { public: - GlobalIndexScanImpl( - const std::string& root_path, const std::shared_ptr& table_schema, - const std::optional& snapshot_id, - const std::optional>>& partitions, - const CoreOptions& options, const std::shared_ptr& pool); + GlobalIndexScanImpl(const std::string& root_path, + const std::shared_ptr& table_schema, const Snapshot& snapshot, + const std::shared_ptr& partitions, + const CoreOptions& options, const std::shared_ptr& pool); Result> CreateRangeScan( const Range& range) override; Result> GetRowRangeList() override; + const Snapshot& GetSnapshot() const { + return snapshot_; + } + private: Status Scan(); + Result>> ParallelScan( + const std::vector& ranges, const std::shared_ptr& predicate, + const std::shared_ptr& executor); + private: bool initialized_ = false; std::shared_ptr pool_; std::string root_path_; std::shared_ptr table_schema_; - std::optional snapshot_id_; - std::optional>> partitions_; + Snapshot snapshot_; + std::shared_ptr partitions_; CoreOptions options_; std::shared_ptr path_factory_; std::vector entries_; diff --git a/src/paimon/core/global_index/indexed_split_impl.h b/src/paimon/core/global_index/indexed_split_impl.h index 9c1bf9ad2..7f1cad75e 100644 --- a/src/paimon/core/global_index/indexed_split_impl.h +++ b/src/paimon/core/global_index/indexed_split_impl.h @@ -89,8 +89,9 @@ class IndexedSplitImpl : public IndexedSplit { } Status Validate() const { - if (row_ranges_.empty()) { - return Status::Invalid("IndexedSplit must have non-empty row ranges"); + if ((row_ranges_.empty() && !data_split_->DataFiles().empty()) || + (!row_ranges_.empty() && data_split_->DataFiles().empty())) { + return Status::Invalid("invalid IndexedSplit: row ranges mismatch data files"); } if (!scores_.empty()) { size_t row_count = 0; diff --git a/src/paimon/core/global_index/indexed_split_test.cpp b/src/paimon/core/global_index/indexed_split_test.cpp index 98a414ec7..b5dae50d8 100644 --- a/src/paimon/core/global_index/indexed_split_test.cpp +++ b/src/paimon/core/global_index/indexed_split_test.cpp @@ -137,25 +137,35 @@ TEST(IndexedSplitTest, TestIndexedSplitWithScore) { } TEST(IndexedSplitTest, TestValidate) { + auto meta = std::make_shared( + "file.orc", 1l, 200l, BinaryRow::EmptyRow(), BinaryRow::EmptyRow(), + SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), 1000l, 1199l, 0, 0, + std::vector>(), Timestamp(0l, 0), 0, nullptr, + FileSource::Append(), std::nullopt, std::nullopt, 1000l, std::nullopt); + + DataSplitImpl::Builder builder( + /*partition=*/BinaryRow::EmptyRow(), + /*bucket=*/0, /*bucket_path=*/ + "data/test_table/bucket-0", std::vector>({meta})); + + auto data_split = std::dynamic_pointer_cast( + builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build().value()); + { std::vector row_ranges = {Range(10, 20), Range(30, 40)}; - IndexedSplitImpl split(/*data_split=*/nullptr, row_ranges); + IndexedSplitImpl split(data_split, row_ranges); ASSERT_OK(split.Validate()); } { std::vector row_ranges = {Range(10, 12), Range(30, 31)}; std::vector scores = {10.01f, 10.11f, 10.21f, -30.01f, -30.11f}; - IndexedSplitImpl split(/*data_split=*/nullptr, row_ranges, scores); + IndexedSplitImpl split(data_split, row_ranges, scores); ASSERT_OK(split.Validate()); } - { - IndexedSplitImpl split(/*data_split=*/nullptr, /*row_ranges=*/std::vector()); - ASSERT_NOK_WITH_MSG(split.Validate(), "IndexedSplit must have non-empty row ranges"); - } { std::vector row_ranges = {Range(10, 12), Range(30, 31)}; std::vector scores = {10.01f, 10.11f, 10.21f, -30.01f}; - IndexedSplitImpl split(/*data_split=*/nullptr, row_ranges, scores); + IndexedSplitImpl split(data_split, row_ranges, scores); ASSERT_NOK_WITH_MSG(split.Validate(), "Scores length does not match row ranges in indexed split."); } diff --git a/src/paimon/core/io/data_file_meta.cpp b/src/paimon/core/io/data_file_meta.cpp index c59192b51..43b1daf8e 100644 --- a/src/paimon/core/io/data_file_meta.cpp +++ b/src/paimon/core/io/data_file_meta.cpp @@ -125,8 +125,8 @@ Result DataFileMeta::CreationTimeEpochMillis() const { } Result> DataFileMeta::ToFileSelection( - const std::vector& row_ranges) const { - if (row_ranges.empty()) { + const std::optional>& row_ranges) const { + if (!row_ranges) { return std::optional(); } PAIMON_ASSIGN_OR_RAISE(int64_t start, NonNullFirstRowId()); @@ -134,7 +134,7 @@ Result> DataFileMeta::ToFileSelection( Range file_range(start, end); RoaringBitmap32 selection; - for (const auto& row_range : row_ranges) { + for (const auto& row_range : row_ranges.value()) { auto intersect_result = Range::Intersection(file_range, row_range); if (intersect_result) { selection.AddRange(static_cast(intersect_result.value().from - start), diff --git a/src/paimon/core/io/data_file_meta.h b/src/paimon/core/io/data_file_meta.h index c28e911ff..28bf62d26 100644 --- a/src/paimon/core/io/data_file_meta.h +++ b/src/paimon/core/io/data_file_meta.h @@ -107,7 +107,7 @@ struct DataFileMeta { // empty row_ranges indicates all rows in the file are needed, return null bitmap Result> ToFileSelection( - const std::vector& row_ranges) const; + const std::optional>& row_ranges) const; static int64_t GetMaxSequenceNumber( const std::vector>& file_metas); diff --git a/src/paimon/core/io/data_file_meta_test.cpp b/src/paimon/core/io/data_file_meta_test.cpp index 19999701e..11f75f6e5 100644 --- a/src/paimon/core/io/data_file_meta_test.cpp +++ b/src/paimon/core/io/data_file_meta_test.cpp @@ -147,17 +147,24 @@ TEST(DataFileMetaTest, TestToFileSelection) { /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/100, /*write_cols=*/std::nullopt); + { + ASSERT_OK_AND_ASSIGN(std::optional result, + file_meta->ToFileSelection(std::nullopt)); + ASSERT_FALSE(result); + } { std::vector ranges = {Range(10l, 20l), Range(105l, 107l), Range(500l, 520l)}; ASSERT_OK_AND_ASSIGN(std::optional result, file_meta->ToFileSelection(ranges)); + ASSERT_TRUE(result); ASSERT_EQ(result.value().ToString(), "{5,6,7}"); } { std::vector ranges = {}; ASSERT_OK_AND_ASSIGN(std::optional result, file_meta->ToFileSelection(ranges)); - ASSERT_FALSE(result); + ASSERT_TRUE(result); + ASSERT_EQ(result.value().ToString(), "{}"); } { std::vector ranges = {Range(100l, 109l)}; diff --git a/src/paimon/core/operation/abstract_file_store_write.cpp b/src/paimon/core/operation/abstract_file_store_write.cpp index e0e191d20..a5d5d9530 100644 --- a/src/paimon/core/operation/abstract_file_store_write.cpp +++ b/src/paimon/core/operation/abstract_file_store_write.cpp @@ -245,8 +245,7 @@ Result AbstractFileStoreWrite::ScanExistingFileMetas( partition_filters.push_back(part_values_map); } auto scan_filter = std::make_shared( - /*predicate=*/nullptr, partition_filters, std::optional(bucket), - /*row_ranges=*/std::vector()); + /*predicate=*/nullptr, partition_filters, std::optional(bucket)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, CreateFileStoreScan(scan_filter)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, diff --git a/src/paimon/core/operation/abstract_split_read.cpp b/src/paimon/core/operation/abstract_split_read.cpp index 59453d128..504011f2b 100644 --- a/src/paimon/core/operation/abstract_split_read.cpp +++ b/src/paimon/core/operation/abstract_split_read.cpp @@ -65,7 +65,7 @@ Result>> AbstractSplitRead::CreateRawFi const BinaryRow& partition, const std::vector>& data_files, const std::shared_ptr& read_schema, const std::shared_ptr& predicate, const std::unordered_map& deletion_file_map, - const std::vector& row_ranges, + const std::optional>& row_ranges, const std::shared_ptr& data_file_path_factory) const { if (data_files.empty()) { return std::vector>(); @@ -170,7 +170,7 @@ Result> AbstractSplitRead::CreateFieldMappingReader const BinaryRow& partition, const ReaderBuilder* reader_builder, const FieldMappingBuilder* field_mapping_builder, const std::unordered_map& deletion_file_map, - const std::vector& row_ranges, + const std::optional>& row_ranges, const std::shared_ptr& data_file_path_factory) const { std::shared_ptr data_schema; if (file_meta->schema_id == context_->GetTableSchema()->Id()) { diff --git a/src/paimon/core/operation/abstract_split_read.h b/src/paimon/core/operation/abstract_split_read.h index df7a18d7a..877b7feaa 100644 --- a/src/paimon/core/operation/abstract_split_read.h +++ b/src/paimon/core/operation/abstract_split_read.h @@ -71,7 +71,7 @@ class AbstractSplitRead : public SplitRead { const std::shared_ptr& read_schema, const std::shared_ptr& predicate, const std::unordered_map& deletion_file_map, - const std::vector& row_ranges, + const std::optional>& row_ranges, const std::shared_ptr& data_file_path_factory) const; static std::unordered_map CreateDeletionFileMap( @@ -88,7 +88,7 @@ class AbstractSplitRead : public SplitRead { const std::shared_ptr& read_schema, const std::shared_ptr& predicate, const std::unordered_map& deletion_file_map, - const std::vector& row_ranges, + const std::optional>& row_ranges, const std::shared_ptr& data_file_path_factory) const = 0; // 1. project write cols to data schema @@ -112,7 +112,7 @@ class AbstractSplitRead : public SplitRead { const BinaryRow& partition, const ReaderBuilder* reader_builder, const FieldMappingBuilder* field_mapping_builder, const std::unordered_map& deletion_file_map, - const std::vector& row_ranges, + const std::optional>& row_ranges, const std::shared_ptr& data_file_path_factory) const; static bool NeedCompleteRowTrackingFields(bool row_tracking_enabled, diff --git a/src/paimon/core/operation/data_evolution_file_store_scan.cpp b/src/paimon/core/operation/data_evolution_file_store_scan.cpp index 1001e5225..07ded98b2 100644 --- a/src/paimon/core/operation/data_evolution_file_store_scan.cpp +++ b/src/paimon/core/operation/data_evolution_file_store_scan.cpp @@ -26,9 +26,9 @@ #include "paimon/common/utils/range_helper.h" namespace paimon { Result DataEvolutionFileStoreScan::FilterEntryByRowRanges( - const ManifestEntry& entry, const std::vector& row_ranges) { + const ManifestEntry& entry, const std::optional>& row_ranges) { // If row ranges is null, all entries should be kept - if (row_ranges.empty()) { + if (!row_ranges) { return true; } // If firstRowId does not exist, keep the entry @@ -41,7 +41,7 @@ Result DataEvolutionFileStoreScan::FilterEntryByRowRanges( int64_t end_row_id = first_row_id.value() + entry.File()->row_count - 1; Range file_range(first_row_id.value(), end_row_id); - for (const auto& row_range : row_ranges) { + for (const auto& row_range : row_ranges.value()) { if (Range::HasIntersection(file_range, row_range)) { return true; } @@ -56,7 +56,7 @@ Result DataEvolutionFileStoreScan::FilterByStats(const ManifestEntry& entr std::vector DataEvolutionFileStoreScan::PostFilterManifests( std::vector&& manifests) const { - if (row_ranges_.empty()) { + if (!row_ranges_) { return std::move(manifests); } std::vector result_metas; @@ -101,9 +101,9 @@ Result> DataEvolutionFileStoreScan::PostFilterManifes return result_entries; } -bool DataEvolutionFileStoreScan::FilterManifestByRowRanges(const ManifestFileMeta& manifest, - const std::vector& row_ranges) { - if (row_ranges.empty()) { +bool DataEvolutionFileStoreScan::FilterManifestByRowRanges( + const ManifestFileMeta& manifest, const std::optional>& row_ranges) { + if (!row_ranges) { return true; } std::optional min = manifest.MinRowId(); @@ -113,7 +113,7 @@ bool DataEvolutionFileStoreScan::FilterManifestByRowRanges(const ManifestFileMet } Range manifest_range(min.value(), max.value()); - for (const auto& range : row_ranges) { + for (const auto& range : row_ranges.value()) { if (Range::HasIntersection(manifest_range, range)) { return true; } diff --git a/src/paimon/core/operation/data_evolution_file_store_scan.h b/src/paimon/core/operation/data_evolution_file_store_scan.h index 45d4a8c77..7c9fc6df9 100644 --- a/src/paimon/core/operation/data_evolution_file_store_scan.h +++ b/src/paimon/core/operation/data_evolution_file_store_scan.h @@ -89,9 +89,9 @@ class DataEvolutionFileStoreScan : public FileStoreScan { Result FilterByStatsWithSameRowId(const std::vector& entries) const; static bool FilterManifestByRowRanges(const ManifestFileMeta& manifest, - const std::vector& row_ranges); + const std::optional>& row_ranges); static Result FilterEntryByRowRanges(const ManifestEntry& entry, - const std::vector& row_ranges); + const std::optional>& row_ranges); static Result> EvolutionStats( const std::vector& entries, const std::shared_ptr& table_schema, diff --git a/src/paimon/core/operation/data_evolution_file_store_scan_test.cpp b/src/paimon/core/operation/data_evolution_file_store_scan_test.cpp index 6d0a10638..c22b40220 100644 --- a/src/paimon/core/operation/data_evolution_file_store_scan_test.cpp +++ b/src/paimon/core/operation/data_evolution_file_store_scan_test.cpp @@ -546,9 +546,15 @@ TEST_F(DataEvolutionFileStoreScanTest, TestFilterEntryByRowRanges) { { // row_ids is null ASSERT_OK_AND_ASSIGN(bool exist, DataEvolutionFileStoreScan::FilterEntryByRowRanges( - entry, /*row_ranges=*/std::vector())); + entry, /*row_ranges=*/std::nullopt)); ASSERT_TRUE(exist); } + { + // row_ids is empty + ASSERT_OK_AND_ASSIGN(bool exist, DataEvolutionFileStoreScan::FilterEntryByRowRanges( + entry, /*row_ranges=*/std::vector())); + ASSERT_FALSE(exist); + } { auto file_without_first_row_id = std::make_shared( "data-0.orc", /*file_size=*/645, @@ -565,27 +571,31 @@ TEST_F(DataEvolutionFileStoreScanTest, TestFilterEntryByRowRanges) { /*bucket=*/0, /*total_buckets=*/1, file_without_first_row_id); // first row id is null - ASSERT_OK_AND_ASSIGN(bool exist, DataEvolutionFileStoreScan::FilterEntryByRowRanges( - entry_without_first_row_id, - /*row_ranges=*/{Range(0l, 0l)})); + ASSERT_OK_AND_ASSIGN( + bool exist, DataEvolutionFileStoreScan::FilterEntryByRowRanges( + entry_without_first_row_id, + /*row_ranges=*/std::optional>({Range(0l, 0l)}))); ASSERT_TRUE(exist); } { ASSERT_OK_AND_ASSIGN(bool exist, DataEvolutionFileStoreScan::FilterEntryByRowRanges( - entry, /*row_ranges=*/{Range(0l, 0l), Range(10l, 10l)})); + entry, /*row_ranges=*/std::optional>( + {Range(0l, 0l), Range(10l, 10l)}))); ASSERT_FALSE(exist); } { ASSERT_OK_AND_ASSIGN(bool exist, DataEvolutionFileStoreScan::FilterEntryByRowRanges( - entry, /*row_ranges=*/{Range(0l, 0l), Range(101l, 101l)})); + entry, /*row_ranges=*/std::optional>( + {Range(0l, 0l), Range(101l, 101l)}))); ASSERT_TRUE(exist); } { ASSERT_OK_AND_ASSIGN(bool exist, DataEvolutionFileStoreScan::FilterEntryByRowRanges( - entry, /*row_ranges=*/{Range(100l, 100l), Range(189l, 189l)})); + entry, /*row_ranges=*/std::optional>( + {Range(100l, 100l), Range(189l, 189l)}))); ASSERT_TRUE(exist); } } @@ -598,11 +608,13 @@ TEST_F(DataEvolutionFileStoreScanTest, TestFilterManifestByRowRanges) { /*schema_id=*/0, /*min_bucket=*/0, /*max_bucket=*/0, /*min_level=*/0, /*max_level=*/0, /*min_row_id=*/10, /*max_row_id=*/20); - ASSERT_TRUE(DataEvolutionFileStoreScan::FilterManifestByRowRanges(manifest1, {})); + ASSERT_TRUE(DataEvolutionFileStoreScan::FilterManifestByRowRanges(manifest1, std::nullopt)); + ASSERT_FALSE( + DataEvolutionFileStoreScan::FilterManifestByRowRanges(manifest1, std::vector())); ASSERT_TRUE(DataEvolutionFileStoreScan::FilterManifestByRowRanges( - manifest1, {Range(0, 15), Range(100, 200)})); + manifest1, std::optional>({Range(0, 15), Range(100, 200)}))); ASSERT_FALSE(DataEvolutionFileStoreScan::FilterManifestByRowRanges( - manifest1, {Range(0, 5), Range(100, 200)})); + manifest1, std::optional>({Range(0, 5), Range(100, 200)}))); auto manifest2 = ManifestFileMeta("manifest-65b0d403-a1bc-4157-b242-bff73c46596d-0", /*file_size=*/2779, @@ -610,6 +622,7 @@ TEST_F(DataEvolutionFileStoreScanTest, TestFilterManifestByRowRanges) { /*schema_id=*/0, /*min_bucket=*/0, /*max_bucket=*/0, /*min_level=*/0, /*max_level=*/0, /*min_row_id=*/std::nullopt, /*max_row_id=*/std::nullopt); - ASSERT_TRUE(DataEvolutionFileStoreScan::FilterManifestByRowRanges(manifest2, {Range(0, 0)})); + ASSERT_TRUE(DataEvolutionFileStoreScan::FilterManifestByRowRanges( + manifest2, std::optional>({Range(0, 0)}))); } } // namespace paimon::test diff --git a/src/paimon/core/operation/data_evolution_split_read.cpp b/src/paimon/core/operation/data_evolution_split_read.cpp index d2572ab28..7a55ec40c 100644 --- a/src/paimon/core/operation/data_evolution_split_read.cpp +++ b/src/paimon/core/operation/data_evolution_split_read.cpp @@ -130,13 +130,14 @@ Result> DataEvolutionSplitRead::CreateReader( return Status::Invalid( "Invalid read schema, read _INDEX_SCORE while split cannot cast to IndexedSplit"); } - return InnerCreateReader(data_split, /*row_ranges=*/{}); + return InnerCreateReader(data_split, /*row_ranges=*/std::nullopt); } return Status::Invalid("Invalid Split, cannot cast to IndexedSplit or DataSplit"); } Result> DataEvolutionSplitRead::InnerCreateReader( - const std::shared_ptr& data_split, const std::vector& row_ranges) const { + const std::shared_ptr& data_split, + const std::optional>& row_ranges) const { auto split_impl = dynamic_cast(data_split.get()); if (split_impl == nullptr) { return Status::Invalid("unexpected error, split cast to impl failed"); @@ -178,7 +179,7 @@ Result> DataEvolutionSplitRead::ApplyIndexAndDvRead const std::shared_ptr& data_schema, const std::shared_ptr& read_schema, const std::shared_ptr& predicate, const std::unordered_map& deletion_file_map, - const std::vector& row_ranges, + const std::optional>& row_ranges, const std::shared_ptr& data_file_path_factory) const { if (!deletion_file_map.empty()) { return Status::Invalid("DataEvolutionSplitRead do not support deletion vector"); @@ -244,7 +245,7 @@ DataEvolutionSplitRead::SplitFieldBunches( Result> DataEvolutionSplitRead::CreateUnionReader( const BinaryRow& partition, const std::vector>& need_merge_files, - const std::vector& row_ranges, + const std::optional>& row_ranges, const std::shared_ptr& data_file_path_factory) const { auto blob_field_to_field_id = [&](const std::shared_ptr& file_meta) -> Result { @@ -266,12 +267,12 @@ Result> DataEvolutionSplitRead::CreateU PAIMON_ASSIGN_OR_RAISE( std::vector> fields_files, SplitFieldBunches(need_merge_files, blob_field_to_field_id, - /*has_row_ranges_selection=*/!row_ranges.empty())); + /*has_row_ranges_selection=*/row_ranges.has_value())); assert(!fields_files.empty()); int64_t row_count = fields_files[0]->RowCount(); PAIMON_ASSIGN_OR_RAISE(int64_t first_row_id, fields_files[0]->Files()[0]->NonNullFirstRowId()); - if (row_ranges.empty()) { + if (!row_ranges) { for (const auto& bunch : fields_files) { if (bunch->RowCount() != row_count) { return Status::Invalid( diff --git a/src/paimon/core/operation/data_evolution_split_read.h b/src/paimon/core/operation/data_evolution_split_read.h index 34ed10b3f..5b8c700a7 100644 --- a/src/paimon/core/operation/data_evolution_split_read.h +++ b/src/paimon/core/operation/data_evolution_split_read.h @@ -77,7 +77,7 @@ class DataEvolutionSplitRead : public AbstractSplitRead { const std::shared_ptr& read_schema, const std::shared_ptr& predicate, const std::unordered_map& deletion_file_map, - const std::vector& row_ranges, + const std::optional>& row_ranges, const std::shared_ptr& data_file_path_factory) const override; private: @@ -129,7 +129,8 @@ class DataEvolutionSplitRead : public AbstractSplitRead { private: Result> InnerCreateReader( - const std::shared_ptr& data_split, const std::vector& row_ranges) const; + const std::shared_ptr& data_split, + const std::optional>& row_ranges) const; static Result>> SplitFieldBunches(const std::vector>& need_merge_files, @@ -145,7 +146,7 @@ class DataEvolutionSplitRead : public AbstractSplitRead { Result> CreateUnionReader( const BinaryRow& partition, const std::vector>& need_merge_files, - const std::vector& row_ranges, + const std::optional>& row_ranges, const std::shared_ptr& data_file_path_factory) const; }; diff --git a/src/paimon/core/operation/file_store_commit_impl.cpp b/src/paimon/core/operation/file_store_commit_impl.cpp index 925adc95e..b6ff9ab1f 100644 --- a/src/paimon/core/operation/file_store_commit_impl.cpp +++ b/src/paimon/core/operation/file_store_commit_impl.cpp @@ -301,8 +301,7 @@ Result FileStoreCommitImpl::GetLastCommitTableRequest() { Result> FileStoreCommitImpl::GetAllFiles( const Snapshot& snapshot, const std::vector>& partitions) { auto scan_filter = std::make_shared(/*predicate=*/nullptr, partitions, - /*bucket_filter=*/std::nullopt, - /*row_ranges=*/std::vector()); + /*bucket_filter=*/std::nullopt); PAIMON_ASSIGN_OR_RAISE( auto scan, AppendOnlyFileStoreScan::Create( snapshot_manager_, schema_manager_, manifest_list_, manifest_file_, @@ -448,8 +447,7 @@ Result> FileStoreCommitImpl::ReadAllEntriesFromChange std::vector> partition_filters(partitions.begin(), partitions.end()); auto scan_filter = std::make_shared(/*predicate=*/nullptr, partition_filters, - /*bucket_filter=*/std::nullopt, - /*row_ranges=*/std::vector()); + /*bucket_filter=*/std::nullopt); PAIMON_ASSIGN_OR_RAISE( auto scan, AppendOnlyFileStoreScan::Create( snapshot_manager_, schema_manager_, manifest_list_, manifest_file_, diff --git a/src/paimon/core/operation/file_store_scan.cpp b/src/paimon/core/operation/file_store_scan.cpp index 00ab9ec41..7199a80fa 100644 --- a/src/paimon/core/operation/file_store_scan.cpp +++ b/src/paimon/core/operation/file_store_scan.cpp @@ -313,7 +313,6 @@ Status FileStoreScan::SplitAndSetFilter(const std::vector& partitio } } bucket_filter_ = scan_filters->GetBucketFilter(); - row_ranges_ = scan_filters->GetRowRanges(); if (!scan_filters->GetPartitionFilters().empty()) { PAIMON_ASSIGN_OR_RAISE( partition_filter_, diff --git a/src/paimon/core/operation/file_store_scan.h b/src/paimon/core/operation/file_store_scan.h index 4ea65b9da..631bf98ba 100644 --- a/src/paimon/core/operation/file_store_scan.h +++ b/src/paimon/core/operation/file_store_scan.h @@ -105,12 +105,17 @@ class FileStoreScan { return this; } - virtual FileStoreScan* EnableValueFilter() { + FileStoreScan* OnlyReadRealBuckets() { + only_read_real_buckets_ = true; return this; } - FileStoreScan* OnlyReadRealBuckets() { - only_read_real_buckets_ = true; + FileStoreScan* WithRowRanges(const std::vector& row_ranges) { + row_ranges_ = row_ranges; + return this; + } + + virtual FileStoreScan* EnableValueFilter() { return this; } @@ -122,6 +127,14 @@ class FileStoreScan { return core_options_; } + std::shared_ptr GetNonPartitionPredicate() const { + return predicates_; + } + + std::shared_ptr GetPartitionPredicate() const { + return partition_filter_; + } + class RawPlan { public: RawPlan(const ScanMode& scan_mode, const std::optional& snapshot, @@ -190,6 +203,11 @@ class FileStoreScan { const std::shared_ptr& arrow_schema, const std::shared_ptr& scan_filters); + static Result> CreatePartitionPredicate( + const std::vector& partition_keys, const std::string& partition_default_name, + const std::shared_ptr& arrow_schema, + const std::vector>& partition_filters); + private: Status ReadManifests(std::optional* snapshot_ptr, std::vector* manifests_ptr) const; @@ -214,18 +232,13 @@ class FileStoreScan { Status ReadManifestFileMeta(const ManifestFileMeta& manifest, std::vector* entries) const; - static Result> CreatePartitionPredicate( - const std::vector& partition_keys, const std::string& partition_default_name, - const std::shared_ptr& arrow_schema, - const std::vector>& partition_filters); - protected: std::shared_ptr pool_; std::shared_ptr schema_manager_; std::shared_ptr predicates_; std::shared_ptr schema_; std::shared_ptr table_schema_; - std::vector row_ranges_; + std::optional> row_ranges_; ScanMode scan_mode_ = ScanMode::ALL; CoreOptions core_options_; diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp index 557fd6042..ac21d0c3d 100644 --- a/src/paimon/core/operation/file_store_write.cpp +++ b/src/paimon/core/operation/file_store_write.cpp @@ -138,8 +138,10 @@ Result> FileStoreWrite::Create(std::unique_ptr trimmed_primary_keys, + schema->TrimmedPrimaryKeys()); PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_key_fields, - schema->TrimmedPrimaryKeyFields()); + schema->GetFields(trimmed_primary_keys)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr key_comparator, FieldsComparator::Create(trimmed_primary_key_fields, options.SequenceFieldSortOrderIsAscending(), diff --git a/src/paimon/core/operation/key_value_file_store_scan_test.cpp b/src/paimon/core/operation/key_value_file_store_scan_test.cpp index ab7c0f913..40c6de693 100644 --- a/src/paimon/core/operation/key_value_file_store_scan_test.cpp +++ b/src/paimon/core/operation/key_value_file_store_scan_test.cpp @@ -132,10 +132,9 @@ TEST_F(KeyValueFileStoreScanTest, TestMaxSequenceNumber) { std::string table_path = paimon::test::GetDataDir() + "orc/pk_table_with_dv_cardinality.db/pk_table_with_dv_cardinality"; std::vector> partition_filters = {{{"f1", "10"}}}; - auto scan_filter = - std::make_shared(/*predicate=*/nullptr, - /*partition_filters=*/partition_filters, - /*bucket_filter=*/0, /*row_ranges=*/std::vector()); + auto scan_filter = std::make_shared(/*predicate=*/nullptr, + /*partition_filters=*/partition_filters, + /*bucket_filter=*/0); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, CreateFileStoreScan(table_path, scan_filter, /*table_schema_id=*/0, /*snapshot_id=*/2)); @@ -156,10 +155,9 @@ TEST_F(KeyValueFileStoreScanTest, TestMaxSequenceNumber) { "orc/pk_table_with_dv_cardinality.db/" "pk_table_with_dv_cardinality"; std::vector> partition_filters = {{{"f1", "10"}}}; - auto scan_filter = - std::make_shared(/*predicate=*/nullptr, - /*partition_filters=*/partition_filters, - /*bucket_filter=*/1, /*row_ranges=*/std::vector()); + auto scan_filter = std::make_shared(/*predicate=*/nullptr, + /*partition_filters=*/partition_filters, + /*bucket_filter=*/1); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, CreateFileStoreScan(table_path, scan_filter, /*table_schema_id=*/0, /*snapshot_id=*/4)); @@ -174,10 +172,9 @@ TEST_F(KeyValueFileStoreScanTest, TestMaxSequenceNumber) { paimon::test::GetDataDir() + "orc/pk_table_with_mor.db/pk_table_with_mor"; std::vector> partition_filters = { {{"p0", "1"}, {"p1", "0"}}}; - auto scan_filter = - std::make_shared(/*predicate=*/nullptr, - /*partition_filters=*/partition_filters, - /*bucket_filter=*/0, /*row_ranges=*/std::vector()); + auto scan_filter = std::make_shared(/*predicate=*/nullptr, + /*partition_filters=*/partition_filters, + /*bucket_filter=*/0); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, CreateFileStoreScan(table_path, scan_filter, /*table_schema_id=*/0, /*snapshot_id=*/1)); @@ -192,10 +189,9 @@ TEST_F(KeyValueFileStoreScanTest, TestMaxSequenceNumber) { paimon::test::GetDataDir() + "orc/pk_table_with_mor.db/pk_table_with_mor"; std::vector> partition_filters = { {{"p0", "0"}, {"p1", "0"}}}; - auto scan_filter = - std::make_shared(/*predicate=*/nullptr, - /*partition_filters=*/partition_filters, - /*bucket_filter=*/0, /*row_ranges=*/std::vector()); + auto scan_filter = std::make_shared(/*predicate=*/nullptr, + /*partition_filters=*/partition_filters, + /*bucket_filter=*/0); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, CreateFileStoreScan(table_path, scan_filter, /*table_schema_id=*/0, /*snapshot_id=*/2)); @@ -209,10 +205,9 @@ TEST_F(KeyValueFileStoreScanTest, TestMaxSequenceNumber) { std::string table_path = paimon::test::GetDataDir() + "orc/pk_table_partial_update.db/pk_table_partial_update"; std::vector> partition_filters = {}; - auto scan_filter = - std::make_shared(/*predicate=*/nullptr, - /*partition_filters=*/partition_filters, - /*bucket_filter=*/0, /*row_ranges=*/std::vector()); + auto scan_filter = std::make_shared(/*predicate=*/nullptr, + /*partition_filters=*/partition_filters, + /*bucket_filter=*/0); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, CreateFileStoreScan(table_path, scan_filter, /*table_schema_id=*/0, /*snapshot_id=*/2)); @@ -238,10 +233,9 @@ TEST_F(KeyValueFileStoreScanTest, TestSplitAndSetKeyValueFilter) { PredicateBuilder::LessThan(/*field_index=*/3, "p0", FieldType::INT, Literal(40)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({not_equal, equal, greater_than, less_than})); - auto scan_filter = - std::make_shared(/*predicate=*/predicate, - /*partition_filters=*/partition_filters, - /*bucket_filter=*/0, /*row_ranges=*/std::vector()); + auto scan_filter = std::make_shared(/*predicate=*/predicate, + /*partition_filters=*/partition_filters, + /*bucket_filter=*/0); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, CreateFileStoreScan(table_path, scan_filter, /*table_schema_id=*/0, /*snapshot_id=*/1)); diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp index 650574763..a180332b4 100644 --- a/src/paimon/core/operation/merge_file_split_read.cpp +++ b/src/paimon/core/operation/merge_file_split_read.cpp @@ -154,7 +154,7 @@ Result> MergeFileSplitRead::ApplyIndexAndDvReaderIf const std::shared_ptr& data_schema, const std::shared_ptr& read_schema, const std::shared_ptr& predicate, const std::unordered_map& deletion_file_map, - const std::vector& ranges, + const std::optional>& ranges, const std::shared_ptr& data_file_path_factory) const { // merge read does not use index PAIMON_UNIQUE_PTR deletion_vector; @@ -276,10 +276,7 @@ Status MergeFileSplitRead::GenerateKeyValueReadSchema( PrimaryKeyTableUtils::CreateSequenceFieldsComparator(value_fields, options)); // 5. complete key fields to all trimmed primary key key_fields.clear(); - for (const auto& key_name : trimmed_key_fields) { - PAIMON_ASSIGN_OR_RAISE(DataField key_field, table_schema.GetField(key_name)); - key_fields.emplace_back(key_field); - } + PAIMON_ASSIGN_OR_RAISE(key_fields, table_schema.GetFields(trimmed_key_fields)); PAIMON_ASSIGN_OR_RAISE( *key_comparator, FieldsComparator::Create(key_fields, /*is_ascending_order=*/true, /*use_view=*/true)); diff --git a/src/paimon/core/operation/merge_file_split_read.h b/src/paimon/core/operation/merge_file_split_read.h index ca01cc1a0..7c077b7dd 100644 --- a/src/paimon/core/operation/merge_file_split_read.h +++ b/src/paimon/core/operation/merge_file_split_read.h @@ -90,7 +90,7 @@ class MergeFileSplitRead : public AbstractSplitRead { const std::shared_ptr& read_schema, const std::shared_ptr& predicate, const std::unordered_map& deletion_file_map, - const std::vector& ranges, + const std::optional>& ranges, const std::shared_ptr& data_file_path_factory) const override; private: diff --git a/src/paimon/core/operation/raw_file_split_read.cpp b/src/paimon/core/operation/raw_file_split_read.cpp index bf30dd6d7..d6da20d1d 100644 --- a/src/paimon/core/operation/raw_file_split_read.cpp +++ b/src/paimon/core/operation/raw_file_split_read.cpp @@ -112,7 +112,7 @@ Result> RawFileSplitRead::ApplyIndexAndDvReaderIfNe const std::shared_ptr& data_schema, const std::shared_ptr& read_schema, const std::shared_ptr& predicate, const std::unordered_map& deletion_file_map, - const std::vector& ranges, + const std::optional>& ranges, const std::shared_ptr& data_file_path_factory) const { std::shared_ptr file_index_result; if (options_.FileIndexReadEnabled()) { diff --git a/src/paimon/core/operation/raw_file_split_read.h b/src/paimon/core/operation/raw_file_split_read.h index 979a575bb..ed0179077 100644 --- a/src/paimon/core/operation/raw_file_split_read.h +++ b/src/paimon/core/operation/raw_file_split_read.h @@ -71,7 +71,7 @@ class RawFileSplitRead : public AbstractSplitRead { const std::shared_ptr& read_schema, const std::shared_ptr& predicate, const std::unordered_map& deletion_file_map, - const std::vector& ranges, + const std::optional>& ranges, const std::shared_ptr& data_file_path_factory) const override; }; diff --git a/src/paimon/core/operation/scan_context.cpp b/src/paimon/core/operation/scan_context.cpp index 6024150ae..aaa2816cb 100644 --- a/src/paimon/core/operation/scan_context.cpp +++ b/src/paimon/core/operation/scan_context.cpp @@ -29,6 +29,7 @@ class Predicate; ScanContext::ScanContext(const std::string& path, bool is_streaming_mode, std::optional limit, const std::shared_ptr& scan_filter, + const std::shared_ptr& global_index_result, const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::map& options) @@ -36,6 +37,7 @@ ScanContext::ScanContext(const std::string& path, bool is_streaming_mode, is_streaming_mode_(is_streaming_mode), limit_(limit), scan_filters_(scan_filter), + global_index_result_(global_index_result), memory_pool_(memory_pool), executor_(executor), options_(options) {} @@ -52,7 +54,7 @@ class ScanContextBuilder::Impl { bucket_filter_ = std::nullopt; partition_filters_.clear(); predicates_.reset(); - row_ranges_.clear(); + global_index_result_.reset(); memory_pool_ = GetDefaultPool(); executor_ = CreateDefaultExecutor(); options_.clear(); @@ -65,7 +67,7 @@ class ScanContextBuilder::Impl { std::optional bucket_filter_; std::vector> partition_filters_; std::shared_ptr predicates_; - std::vector row_ranges_; + std::shared_ptr global_index_result_; std::shared_ptr memory_pool_ = GetDefaultPool(); std::shared_ptr executor_ = CreateDefaultExecutor(); std::map options_; @@ -102,8 +104,9 @@ ScanContextBuilder& ScanContextBuilder::SetPredicate(const std::shared_ptr& row_ranges) { - impl_->row_ranges_ = row_ranges; +ScanContextBuilder& ScanContextBuilder::SetGlobalIndexResult( + const std::shared_ptr& global_index_result) { + impl_->global_index_result_ = global_index_result; return *this; } @@ -138,8 +141,8 @@ Result> ScanContextBuilder::Finish() { auto ctx = std::make_unique( impl_->path_, impl_->is_streaming_mode_, impl_->limit_, std::make_shared(impl_->predicates_, impl_->partition_filters_, - impl_->bucket_filter_, impl_->row_ranges_), - impl_->memory_pool_, impl_->executor_, impl_->options_); + impl_->bucket_filter_), + impl_->global_index_result_, impl_->memory_pool_, impl_->executor_, impl_->options_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/scan_context_test.cpp b/src/paimon/core/operation/scan_context_test.cpp index 9bec76985..3f7c52e4c 100644 --- a/src/paimon/core/operation/scan_context_test.cpp +++ b/src/paimon/core/operation/scan_context_test.cpp @@ -18,6 +18,7 @@ #include "gtest/gtest.h" #include "paimon/defs.h" +#include "paimon/global_index/bitmap_global_index_result.h" #include "paimon/predicate/predicate_builder.h" #include "paimon/status.h" #include "paimon/testing/utils/testharness.h" @@ -35,7 +36,7 @@ TEST(ScanContextTest, TestSimple) { ASSERT_FALSE(ctx->GetScanFilters()->GetBucketFilter()); ASSERT_FALSE(ctx->GetScanFilters()->GetPredicate()); ASSERT_TRUE(ctx->GetScanFilters()->GetPartitionFilters().empty()); - ASSERT_TRUE(ctx->GetScanFilters()->GetRowRanges().empty()); + ASSERT_FALSE(ctx->GetGlobalIndexResult()); } TEST(ScanContextTest, TestSetFilter) { @@ -47,7 +48,8 @@ TEST(ScanContextTest, TestSetFilter) { PredicateBuilder::IsNull(/*field_index=*/2, /*field_name=*/"f2", FieldType::INT); builder.SetPredicate(predicate); std::vector row_ranges = {Range(1, 2), Range(4, 5)}; - builder.SetRowRanges(row_ranges); + auto global_index_result = BitmapGlobalIndexResult::FromRanges(row_ranges); + builder.SetGlobalIndexResult(global_index_result); builder.SetLimit(1000); builder.AddOption("key", "value"); builder.WithStreamingMode(true); @@ -59,7 +61,7 @@ TEST(ScanContextTest, TestSetFilter) { ASSERT_EQ(10, ctx->GetScanFilters()->GetBucketFilter()); ASSERT_EQ(*predicate, *(ctx->GetScanFilters()->GetPredicate())); ASSERT_EQ(partition_filters, ctx->GetScanFilters()->GetPartitionFilters()); - ASSERT_EQ(row_ranges, ctx->GetScanFilters()->GetRowRanges()); + ASSERT_EQ("{1,2,4,5}", ctx->GetGlobalIndexResult()->ToString()); std::map expected_options = {{"key", "value"}}; ASSERT_EQ(expected_options, ctx->GetOptions()); } diff --git a/src/paimon/core/schema/table_schema.cpp b/src/paimon/core/schema/table_schema.cpp index bb92fd297..e5ee56d3f 100644 --- a/src/paimon/core/schema/table_schema.cpp +++ b/src/paimon/core/schema/table_schema.cpp @@ -197,16 +197,12 @@ Result TableSchema::GetField(int32_t field_id) const { fmt::format("Get field with id {} failed: not exist in table schema", field_id)); } -Result> TableSchema::TrimmedPrimaryKeyFields() const { - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, TrimmedPrimaryKeys()); +Result> TableSchema::GetFields( + const std::vector& field_names) const { std::vector data_fields; - for (const auto& trimmed_primary_key : trimmed_primary_keys) { - for (const auto& field : fields_) { - if (field.Name() == trimmed_primary_key) { - data_fields.emplace_back(field); - break; - } - } + for (const auto& name : field_names) { + PAIMON_ASSIGN_OR_RAISE(DataField field, GetField(name)); + data_fields.emplace_back(field); } return data_fields; } diff --git a/src/paimon/core/schema/table_schema.h b/src/paimon/core/schema/table_schema.h index d75c1e86c..25e87322a 100644 --- a/src/paimon/core/schema/table_schema.h +++ b/src/paimon/core/schema/table_schema.h @@ -84,12 +84,12 @@ class TableSchema : public Jsonizable { const std::vector& Fields() const { return fields_; } - Result> TrimmedPrimaryKeyFields() const; Result GetField(const std::string& field_name) const; Result GetField(int32_t field_id) const; + Result> GetFields(const std::vector& field_names) const; Result> TrimmedPrimaryKeys() const; std::optional Comment() const { diff --git a/src/paimon/core/schema/table_schema_test.cpp b/src/paimon/core/schema/table_schema_test.cpp index 13d29705c..861dbc593 100644 --- a/src/paimon/core/schema/table_schema_test.cpp +++ b/src/paimon/core/schema/table_schema_test.cpp @@ -213,8 +213,10 @@ TEST_F(TableSchemaTest, TestDeserializePkTableSchema) { ASSERT_EQ(*result_schema, expected_schema); ASSERT_EQ(std::vector({"f0", "f1", "f2", "f3"}), result_schema->FieldNames()); + ASSERT_OK_AND_ASSIGN(auto trimmed_primary_keys, result_schema->TrimmedPrimaryKeys()); + ASSERT_EQ(trimmed_primary_keys, std::vector({"f0", "f2"})); ASSERT_OK_AND_ASSIGN(std::vector trimmed_primary_key_fields, - result_schema->TrimmedPrimaryKeyFields()); + result_schema->GetFields(trimmed_primary_keys)); ASSERT_EQ(2, trimmed_primary_key_fields.size()); ASSERT_EQ(trimmed_primary_key_fields[0], field1); ASSERT_EQ(trimmed_primary_key_fields[1], field3); diff --git a/src/paimon/core/table/source/data_evolution_batch_scan.cpp b/src/paimon/core/table/source/data_evolution_batch_scan.cpp new file mode 100644 index 000000000..1a8b59078 --- /dev/null +++ b/src/paimon/core/table/source/data_evolution_batch_scan.cpp @@ -0,0 +1,150 @@ +/* + * Copyright 2025-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/table/source/data_evolution_batch_scan.h" + +#include "paimon/core/global_index/global_index_scan_impl.h" +#include "paimon/core/global_index/indexed_split_impl.h" +#include "paimon/core/table/source/data_split_impl.h" +#include "paimon/global_index/bitmap_global_index_result.h" +#include "paimon/global_index/global_index_scan.h" + +namespace paimon { +DataEvolutionBatchScan::DataEvolutionBatchScan( + const std::string& table_path, const std::shared_ptr& snapshot_reader, + std::unique_ptr&& batch_scan, + const std::shared_ptr& global_index_result, const CoreOptions& core_options, + const std::shared_ptr& pool, const std::shared_ptr& executor) + : AbstractTableScan(core_options, snapshot_reader), + pool_(pool), + table_path_(table_path), + batch_scan_(std::move(batch_scan)), + global_index_result_(global_index_result), + executor_(executor) {} + +Result> DataEvolutionBatchScan::CreatePlan() { + std::optional> row_ranges; + if (!global_index_result_) { + PAIMON_ASSIGN_OR_RAISE(std::optional> index_result, + EvalGlobalIndex()); + if (index_result) { + PAIMON_ASSIGN_OR_RAISE(row_ranges, index_result.value()->ToRanges()); + } + } else { + PAIMON_ASSIGN_OR_RAISE(row_ranges, global_index_result_->ToRanges()); + } + if (!row_ranges) { + return batch_scan_->CreatePlan(); + } + batch_scan_->WithRowRanges(row_ranges.value()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_plan, batch_scan_->CreatePlan()); + std::map id_to_score; + if (auto topk_result = std::dynamic_pointer_cast(global_index_result_)) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr topk_iter, + topk_result->CreateTopKIterator()); + while (topk_iter->HasNext()) { + auto [id, score] = topk_iter->NextWithScore(); + id_to_score[id] = score; + } + } + return WrapToIndexedSplits(data_plan, row_ranges.value(), id_to_score); +} + +Result> DataEvolutionBatchScan::WrapToIndexedSplits( + const std::shared_ptr& data_plan, const std::vector& row_ranges, + const std::map& id_to_score) const { + std::vector sorted_row_ranges = + Range::SortAndMergeOverlap(row_ranges, /*adjacent=*/true); + std::vector> indexed_splits; + for (const auto& split : data_plan->Splits()) { + auto data_split = std::dynamic_pointer_cast(split); + if (!data_split) { + return Status::Invalid("Cannot cast split to DataSplit when create IndexedSplit"); + } + std::vector file_ranges; + file_ranges.reserve(data_split->DataFiles().size()); + for (const auto& meta : data_split->DataFiles()) { + PAIMON_ASSIGN_OR_RAISE(int64_t first_row_id, meta->NonNullFirstRowId()); + file_ranges.emplace_back(first_row_id, first_row_id + meta->row_count - 1); + } + auto sorted_file_ranges = Range::SortAndMergeOverlap(file_ranges, /*adjacent=*/true); + std::vector expected = Range::And(sorted_file_ranges, sorted_row_ranges); + std::vector scores; + if (!id_to_score.empty()) { + for (const auto& range : expected) { + for (int64_t i = range.from; i <= range.to; i++) { + auto iter = id_to_score.find(i); + if (iter != id_to_score.end()) { + scores.push_back(iter->second); + } else { + return Status::Invalid(fmt::format("cannot find score for row {}", i)); + } + } + } + } + indexed_splits.push_back(std::make_shared(data_split, expected, scores)); + } + return std::make_shared(data_plan->SnapshotId(), indexed_splits); +} + +Result>> DataEvolutionBatchScan::EvalGlobalIndex() + const { + auto predicate = batch_scan_->GetNonPartitionPredicate(); + if (!predicate) { + return std::optional>(); + } + if (!core_options_.GlobalIndexEnabled()) { + return std::optional>(); + } + auto partition_filter = batch_scan_->GetPartitionPredicate(); + // TODO(lisizhuo.lsz): support time travel + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr index_scan, + GlobalIndexScan::Create(table_path_, core_options_.GetScanSnapshotId(), partition_filter, + core_options_.ToMap(), core_options_.GetFileSystem(), pool_)); + auto index_scan_impl = dynamic_cast(index_scan.get()); + if (!index_scan_impl) { + return Status::Invalid("invalid GlobalIndexScan, cannot cast to GlobalIndexScanImpl"); + } + PAIMON_ASSIGN_OR_RAISE(std::vector indexed_row_ranges, index_scan->GetRowRangeList()); + if (indexed_row_ranges.empty()) { + return std::optional>(); + } + const auto& snapshot = index_scan_impl->GetSnapshot(); + const std::optional& next_row_id = snapshot.NextRowId(); + if (!next_row_id) { + return Status::Invalid("invalid snapshot, next row id is null"); + } + + std::vector non_indexed_row_ranges = + Range(0, next_row_id.value() - 1).Exclude(indexed_row_ranges); + PAIMON_ASSIGN_OR_RAISE(std::optional> index_result, + index_scan_impl->ParallelScan(indexed_row_ranges, predicate, executor_)); + if (!index_result) { + return std::optional>(); + } + auto index_result_value = std::move(index_result).value(); + if (!non_indexed_row_ranges.empty()) { + for (const auto& range : non_indexed_row_ranges) { + PAIMON_ASSIGN_OR_RAISE( + index_result_value, + index_result_value->Or(BitmapGlobalIndexResult::FromRanges({range}))); + } + } + return std::optional>(index_result_value); +} + +} // namespace paimon diff --git a/src/paimon/core/table/source/data_evolution_batch_scan.h b/src/paimon/core/table/source/data_evolution_batch_scan.h new file mode 100644 index 000000000..8367209eb --- /dev/null +++ b/src/paimon/core/table/source/data_evolution_batch_scan.h @@ -0,0 +1,54 @@ +/* + * Copyright 2025-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include + +#include "paimon/core/table/source/abstract_table_scan.h" +#include "paimon/core/table/source/data_table_batch_scan.h" +#include "paimon/result.h" +#include "paimon/utils/range.h" + +namespace paimon { +class DataEvolutionBatchScan : public AbstractTableScan { + public: + DataEvolutionBatchScan(const std::string& table_path, + const std::shared_ptr& snapshot_reader, + std::unique_ptr&& batch_scan, + const std::shared_ptr& global_index_result, + const CoreOptions& core_options, const std::shared_ptr& pool, + const std::shared_ptr& executor); + + Result> CreatePlan() override; + + private: + Result> WrapToIndexedSplits( + const std::shared_ptr& data_plan, const std::vector& row_ranges, + const std::map& id_to_score) const; + Result>> EvalGlobalIndex() const; + + private: + std::shared_ptr pool_; + std::string table_path_; + std::unique_ptr batch_scan_; + std::shared_ptr global_index_result_; + std::shared_ptr executor_; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/source/data_table_batch_scan.h b/src/paimon/core/table/source/data_table_batch_scan.h index db18c1ddb..405b784d6 100644 --- a/src/paimon/core/table/source/data_table_batch_scan.h +++ b/src/paimon/core/table/source/data_table_batch_scan.h @@ -37,6 +37,18 @@ class DataTableBatchScan : public AbstractTableScan { Result> CreatePlan() override; + std::shared_ptr GetNonPartitionPredicate() const { + return snapshot_reader_->GetNonPartitionPredicate(); + } + std::shared_ptr GetPartitionPredicate() const { + return snapshot_reader_->GetPartitionPredicate(); + } + + DataTableBatchScan* WithRowRanges(const std::vector& row_ranges) { + snapshot_reader_->WithRowRanges(row_ranges); + return this; + } + private: Result> ApplyPushDownLimit( const std::shared_ptr& scan_result) const; diff --git a/src/paimon/core/table/source/snapshot/snapshot_reader.h b/src/paimon/core/table/source/snapshot/snapshot_reader.h index 03a7ebca0..b590cd077 100644 --- a/src/paimon/core/table/source/snapshot/snapshot_reader.h +++ b/src/paimon/core/table/source/snapshot/snapshot_reader.h @@ -80,10 +80,23 @@ class SnapshotReader { return this; } + SnapshotReader* WithRowRanges(const std::vector& row_ranges) { + scan_->WithRowRanges(row_ranges); + return this; + } + const std::shared_ptr& GetSnapshotManager() const { return scan_->GetSnapshotManager(); } + std::shared_ptr GetNonPartitionPredicate() const { + return scan_->GetNonPartitionPredicate(); + } + + std::shared_ptr GetPartitionPredicate() const { + return scan_->GetPartitionPredicate(); + } + /// Get splits from `FileKind::ADD` files. Result> Read() const; diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp index e5347437b..62af3056b 100644 --- a/src/paimon/core/table/source/table_scan.cpp +++ b/src/paimon/core/table/source/table_scan.cpp @@ -38,7 +38,9 @@ #include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/bucket_mode.h" +#include "paimon/core/table/source/abstract_table_scan.h" #include "paimon/core/table/source/append_only_split_generator.h" +#include "paimon/core/table/source/data_evolution_batch_scan.h" #include "paimon/core/table/source/data_evolution_split_generator.h" #include "paimon/core/table/source/data_table_batch_scan.h" #include "paimon/core/table/source/data_table_stream_scan.h" @@ -119,8 +121,10 @@ class TableScanImpl { source_split_target_size, source_split_open_file_cost, bucket_mode); } else { // TODO(liancheng.lsz): support evolution + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, + table_schema->TrimmedPrimaryKeys()); PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_pk_fields, - table_schema->TrimmedPrimaryKeyFields()); + table_schema->GetFields(trimmed_primary_keys)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr key_comparator, FieldsComparator::Create(trimmed_pk_fields, /*is_ascending_order=*/true, @@ -222,8 +226,15 @@ Result> TableScan::Create(std::unique_ptrIsStreamingMode()) { return std::make_unique(core_options, snapshot_reader); } - return std::make_unique(/*pk_table=*/!table_schema->PrimaryKeys().empty(), - core_options, snapshot_reader, context->GetLimit()); + auto batch_scan = + std::make_unique(/*pk_table=*/!table_schema->PrimaryKeys().empty(), + core_options, snapshot_reader, context->GetLimit()); + if (!core_options.DataEvolutionEnabled()) { + return batch_scan; + } + return std::make_unique( + context->GetPath(), snapshot_reader, std::move(batch_scan), context->GetGlobalIndexResult(), + core_options, context->GetMemoryPool(), context->GetExecutor()); } } // namespace paimon diff --git a/src/paimon/global_index/lumina/lumina_global_index.h b/src/paimon/global_index/lumina/lumina_global_index.h index 28d5e2a37..c0be6e6a0 100644 --- a/src/paimon/global_index/lumina/lumina_global_index.h +++ b/src/paimon/global_index/lumina/lumina_global_index.h @@ -94,58 +94,58 @@ class LuminaIndexReader : public GlobalIndexReader { const std::shared_ptr& predicate) override; Result> VisitIsNotNull() override { - return BitmapGlobalIndexResult::FromRange(range_); + return BitmapGlobalIndexResult::FromRanges({range_}); } Result> VisitIsNull() override { - return BitmapGlobalIndexResult::FromRange(range_); + return BitmapGlobalIndexResult::FromRanges({range_}); } Result> VisitEqual(const Literal& literal) override { - return BitmapGlobalIndexResult::FromRange(range_); + return BitmapGlobalIndexResult::FromRanges({range_}); } Result> VisitNotEqual(const Literal& literal) override { - return BitmapGlobalIndexResult::FromRange(range_); + return BitmapGlobalIndexResult::FromRanges({range_}); } Result> VisitLessThan(const Literal& literal) override { - return BitmapGlobalIndexResult::FromRange(range_); + return BitmapGlobalIndexResult::FromRanges({range_}); } Result> VisitLessOrEqual(const Literal& literal) override { - return BitmapGlobalIndexResult::FromRange(range_); + return BitmapGlobalIndexResult::FromRanges({range_}); } Result> VisitGreaterThan(const Literal& literal) override { - return BitmapGlobalIndexResult::FromRange(range_); + return BitmapGlobalIndexResult::FromRanges({range_}); } Result> VisitGreaterOrEqual( const Literal& literal) override { - return BitmapGlobalIndexResult::FromRange(range_); + return BitmapGlobalIndexResult::FromRanges({range_}); } Result> VisitIn( const std::vector& literals) override { - return BitmapGlobalIndexResult::FromRange(range_); + return BitmapGlobalIndexResult::FromRanges({range_}); } Result> VisitNotIn( const std::vector& literals) override { - return BitmapGlobalIndexResult::FromRange(range_); + return BitmapGlobalIndexResult::FromRanges({range_}); } Result> VisitStartsWith(const Literal& prefix) override { - return BitmapGlobalIndexResult::FromRange(range_); + return BitmapGlobalIndexResult::FromRanges({range_}); } Result> VisitEndsWith(const Literal& suffix) override { - return BitmapGlobalIndexResult::FromRange(range_); + return BitmapGlobalIndexResult::FromRanges({range_}); } Result> VisitContains(const Literal& literal) override { - return BitmapGlobalIndexResult::FromRange(range_); + return BitmapGlobalIndexResult::FromRanges({range_}); } private: diff --git a/src/paimon/testing/utils/data_generator.cpp b/src/paimon/testing/utils/data_generator.cpp index 1b4b44910..1cbd66659 100644 --- a/src/paimon/testing/utils/data_generator.cpp +++ b/src/paimon/testing/utils/data_generator.cpp @@ -110,16 +110,6 @@ Result DataGenerator::ExtractPartialRow(const BinaryRow& binary_row, return partial_row; } -Result> DataGenerator::GetDataFields( - const std::vector& field_names) { - std::vector fields; - for (const auto& field_name : field_names) { - PAIMON_ASSIGN_OR_RAISE(auto field, table_schema_->GetField(field_name)); - fields.push_back(field); - } - return fields; -} - Status DataGenerator::AppendValue(const BinaryRow& row, int32_t field_id, const std::shared_ptr& type, arrow::StructBuilder* struct_builder) { @@ -232,8 +222,9 @@ Result>> DataGenerator::SplitArrayByPar const std::vector& binary_rows) { auto fields = table_schema_->Fields(); auto partition_keys = table_schema_->PartitionKeys(); - PAIMON_ASSIGN_OR_RAISE(auto partition_fields, GetDataFields(partition_keys)); - PAIMON_ASSIGN_OR_RAISE(auto bucket_fields, GetDataFields(table_schema_->BucketKeys())); + PAIMON_ASSIGN_OR_RAISE(auto partition_fields, table_schema_->GetFields(partition_keys)); + PAIMON_ASSIGN_OR_RAISE(auto bucket_fields, + table_schema_->GetFields(table_schema_->BucketKeys())); int32_t num_buckets = table_schema_->NumBuckets(); // map: {partition_map, bucket_id} -> arrow::StructBuilder std::map, int32_t>, diff --git a/src/paimon/testing/utils/data_generator.h b/src/paimon/testing/utils/data_generator.h index c74777c60..d96f3d072 100644 --- a/src/paimon/testing/utils/data_generator.h +++ b/src/paimon/testing/utils/data_generator.h @@ -55,8 +55,6 @@ class DataGenerator { Result ExtractPartialRow(const BinaryRow& binary_row, const std::vector& partition_fields); - Result> GetDataFields(const std::vector& field_names); - static Status WriteBinaryRow(const BinaryRow& src_row, int32_t src_field_id, const std::shared_ptr& src_type, int32_t target_field_id, BinaryRowWriter* target_row_writer); diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp index 4b0efbd6a..727690a1a 100644 --- a/test/inte/blob_table_inte_test.cpp +++ b/test/inte/blob_table_inte_test.cpp @@ -52,6 +52,7 @@ #include "paimon/file_store_write.h" #include "paimon/fs/file_system.h" #include "paimon/fs/local/local_file_system.h" +#include "paimon/global_index/bitmap_global_index_result.h" #include "paimon/memory/memory_pool.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate_builder.h" @@ -158,43 +159,17 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter return file_store_commit->Commit(commit_msgs); } - Result>> CreateReadSplit( - const std::vector>& splits, - const std::vector& row_ranges) const { - if (row_ranges.empty()) { - return splits; - } - // TODO(xinyu.lxy): mv to DataEvolutionBatchScan - std::vector sorted_row_ranges = - Range::SortAndMergeOverlap(row_ranges, /*adjacent=*/true); - std::vector> indexed_splits; - indexed_splits.reserve(splits.size()); - for (const auto& split : splits) { - auto data_split = std::dynamic_pointer_cast(split); - if (!data_split) { - return Status::Invalid("Cannot cast split to DataSplit when create IndexedSplit"); - } - std::vector file_ranges; - file_ranges.reserve(data_split->DataFiles().size()); - for (const auto& meta : data_split->DataFiles()) { - PAIMON_ASSIGN_OR_RAISE(int64_t first_row_id, meta->NonNullFirstRowId()); - file_ranges.emplace_back(first_row_id, first_row_id + meta->row_count - 1); - } - auto sorted_file_ranges = Range::SortAndMergeOverlap(file_ranges, /*adjacent=*/true); - std::vector expected = Range::And(sorted_file_ranges, sorted_row_ranges); - // TODO(xinyu.lxy): add scores - indexed_splits.push_back(std::make_shared(data_split, expected)); - } - return indexed_splits; - } - Status ScanAndRead(const std::string& table_path, const std::vector& read_schema, const std::shared_ptr& expected_array, const std::shared_ptr& predicate = nullptr, const std::vector& row_ranges = {}) const { // scan ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetPredicate(predicate).SetRowRanges(row_ranges); + scan_context_builder.SetPredicate(predicate); + if (!row_ranges.empty()) { + auto global_index_result = BitmapGlobalIndexResult::FromRanges(row_ranges); + scan_context_builder.SetGlobalIndexResult(global_index_result); + } PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); @@ -209,8 +184,7 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); - PAIMON_ASSIGN_OR_RAISE(auto read_splits, CreateReadSplit(splits, row_ranges)); - PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(read_splits)); + PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); PAIMON_ASSIGN_OR_RAISE(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); diff --git a/test/inte/data_evolution_table_test.cpp b/test/inte/data_evolution_table_test.cpp index 25cd5b4e5..247e96336 100644 --- a/test/inte/data_evolution_table_test.cpp +++ b/test/inte/data_evolution_table_test.cpp @@ -24,6 +24,7 @@ #include "paimon/core/table/source/data_split_impl.h" #include "paimon/defs.h" #include "paimon/fs/file_system.h" +#include "paimon/global_index/bitmap_global_index_result.h" #include "paimon/global_index/indexed_split.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate_builder.h" @@ -122,36 +123,6 @@ class DataEvolutionTableTest : public ::testing::Test, return file_store_commit->Commit(commit_msgs); } - Result>> CreateReadSplit( - const std::vector>& splits, - const std::vector& row_ranges) const { - if (row_ranges.empty()) { - return splits; - } - // TODO(xinyu.lxy): mv to DataEvolutionBatchScan - std::vector sorted_row_ranges = - Range::SortAndMergeOverlap(row_ranges, /*adjacent=*/true); - std::vector> indexed_splits; - indexed_splits.reserve(splits.size()); - for (const auto& split : splits) { - auto data_split = std::dynamic_pointer_cast(split); - if (!data_split) { - return Status::Invalid("Cannot cast split to DataSplit when create IndexedSplit"); - } - std::vector file_ranges; - file_ranges.reserve(data_split->DataFiles().size()); - for (const auto& meta : data_split->DataFiles()) { - PAIMON_ASSIGN_OR_RAISE(int64_t first_row_id, meta->NonNullFirstRowId()); - file_ranges.emplace_back(first_row_id, first_row_id + meta->row_count - 1); - } - auto sorted_file_ranges = Range::SortAndMergeOverlap(file_ranges, /*adjacent=*/true); - std::vector expected = Range::And(sorted_file_ranges, sorted_row_ranges); - // TODO(xinyu.lxy): add scores - indexed_splits.push_back(std::make_shared(data_split, expected)); - } - return indexed_splits; - } - Status ScanAndRead(const std::string& table_path, const std::vector& read_schema, const std::shared_ptr& expected_array, const std::shared_ptr& predicate = nullptr, @@ -159,7 +130,11 @@ class DataEvolutionTableTest : public ::testing::Test, bool check_scan_plan_when_empty_result = true) const { // scan ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetPredicate(predicate).SetRowRanges(row_ranges); + scan_context_builder.SetPredicate(predicate); + if (!row_ranges.empty()) { + auto global_index_result = BitmapGlobalIndexResult::FromRanges(row_ranges); + scan_context_builder.SetGlobalIndexResult(global_index_result); + } PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); @@ -176,8 +151,7 @@ class DataEvolutionTableTest : public ::testing::Test, PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); - PAIMON_ASSIGN_OR_RAISE(auto read_splits, CreateReadSplit(splits, row_ranges)); - PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(read_splits)); + PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); PAIMON_ASSIGN_OR_RAISE(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); @@ -222,7 +196,11 @@ class DataEvolutionTableTest : public ::testing::Test, const std::vector& expected_row_counts) { ASSERT_EQ(expected_first_row_ids.size(), expected_row_counts.size()); ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetPredicate(predicate).SetRowRanges(row_ranges); + scan_context_builder.SetPredicate(predicate); + if (!row_ranges.empty()) { + auto global_index_result = BitmapGlobalIndexResult::FromRanges(row_ranges); + scan_context_builder.SetGlobalIndexResult(global_index_result); + } ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -232,7 +210,12 @@ class DataEvolutionTableTest : public ::testing::Test, return; } ASSERT_EQ(result_splits.size(), 1); - auto data_split = std::dynamic_pointer_cast(result_splits[0]); + std::shared_ptr data_split; + if (auto indexed_split = std::dynamic_pointer_cast(result_splits[0])) { + data_split = std::dynamic_pointer_cast(indexed_split->GetDataSplit()); + } else { + data_split = std::dynamic_pointer_cast(result_splits[0]); + } ASSERT_TRUE(data_split); std::vector> result_first_row_ids; std::vector result_row_counts; @@ -750,19 +733,6 @@ TEST_P(DataEvolutionTableTest, TestOnlyRowTrackingEnabled) { ASSERT_OK_AND_ASSIGN(auto commit_msgs, WriteArray(table_path, write_cols0, src_array0)); ASSERT_OK(Commit(table_path, commit_msgs)); - { - // test with row ids, as only data evolution mode support read with row ranges - std::vector row_ranges = {Range(1l, 1l)}; - auto expected_array = std::dynamic_pointer_cast( - arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_), R"([ - [2, "c", "d"] - ])") - .ValueOrDie()); - ASSERT_NOK_WITH_MSG(ScanAndRead(table_path, schema->field_names(), expected_array, - /*predicate=*/nullptr, - /*row_ranges=*/row_ranges), - "unexpected error, split cast to impl failed"); - } if (GetParam() != "lance") { // read with row tracking auto expected_row_tracking_array = std::dynamic_pointer_cast( @@ -1326,32 +1296,16 @@ TEST_P(DataEvolutionTableTest, TestReadCompactFiles) { std::shared_ptr arrow_data_type = DataField::ConvertDataFieldsToArrowStructType(read_fields); - { - auto expected_array = std::dynamic_pointer_cast( - arrow::ipc::internal::json::ArrayFromJSON(arrow_data_type, R"([ + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow_data_type, R"([ ["Lily", 2, 12, 2.1, 0, 1], ["Alice", 3, 13, 3.1, 1, 1], ["Bob", 4, 14, 4.1, 2, 2], ["David", 5, 15, 5.1, 3, 2] ])") - .ValueOrDie()); - ASSERT_OK(ScanAndRead(table_path, arrow::schema(arrow_data_type->fields())->field_names(), - expected_array)); - } - { - // test with row ids - std::vector row_ranges = {Range(1l, 1l)}; - // as after compact, first row id in data file meta is null, read will fail - CheckScanResult(table_path, /*predicate=*/nullptr, /*row_ranges=*/row_ranges, - /*expected_first_row_ids=*/{std::nullopt}, /*expected_row_counts=*/{4}); - auto read_status = - ScanAndRead(table_path, arrow::schema(arrow_data_type->fields())->field_names(), - /*expected_array=*/nullptr, - /*predicate=*/nullptr, - /*row_ranges=*/row_ranges, /*check_scan_plan_when_empty_result=*/false); - ASSERT_NOK_WITH_MSG(read_status, "First row id of "); - ASSERT_NOK_WITH_MSG(read_status, "should not be null"); - } + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, arrow::schema(arrow_data_type->fields())->field_names(), + expected_array)); } TEST_P(DataEvolutionTableTest, TestReadTableWithDenseStats) { diff --git a/test/inte/global_index_test.cpp b/test/inte/global_index_test.cpp index 23adf2f08..d16341fb9 100644 --- a/test/inte/global_index_test.cpp +++ b/test/inte/global_index_test.cpp @@ -23,6 +23,8 @@ #include "paimon/core/table/source/data_split_impl.h" #include "paimon/defs.h" #include "paimon/fs/file_system.h" +#include "paimon/global_index/bitmap_global_index_result.h" +#include "paimon/global_index/bitmap_topk_global_index_result.h" #include "paimon/global_index/global_index_scan.h" #include "paimon/global_index/row_range_global_index_writer.h" #include "paimon/predicate/literal.h" @@ -123,70 +125,64 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter return std::dynamic_pointer_cast(result_plan->Splits()[0]); } - Result>> CreateReadSplit( - const std::vector>& splits, const std::vector& row_ranges, - std::map id_to_score) const { - if (row_ranges.empty()) { - return splits; - } - // TODO(xinyu.lxy): mv to DataEvolutionBatchScan - std::vector sorted_row_ranges = - Range::SortAndMergeOverlap(row_ranges, /*adjacent=*/true); - std::vector> indexed_splits; - indexed_splits.reserve(splits.size()); - for (const auto& split : splits) { - auto data_split = std::dynamic_pointer_cast(split); - if (!data_split) { - return Status::Invalid("Cannot cast split to DataSplit when create IndexedSplit"); - } - std::vector file_ranges; - file_ranges.reserve(data_split->DataFiles().size()); - for (const auto& meta : data_split->DataFiles()) { - PAIMON_ASSIGN_OR_RAISE(int64_t first_row_id, meta->NonNullFirstRowId()); - file_ranges.emplace_back(first_row_id, first_row_id + meta->row_count - 1); - } - auto sorted_file_ranges = Range::SortAndMergeOverlap(file_ranges, /*adjacent=*/true); - std::vector expected = Range::And(sorted_file_ranges, sorted_row_ranges); - std::vector scores; - if (!id_to_score.empty()) { - for (const auto& range : expected) { - for (int64_t i = range.from; i <= range.to; i++) { - scores.push_back(id_to_score[i]); - } - } - } - indexed_splits.push_back( - std::make_shared(data_split, expected, scores)); - } - return indexed_splits; + Status WriteIndex(const std::string& table_path, + const std::vector>& partition_filters, + const std::string& index_field_name, const std::string& index_type, + const std::map& options, const Range& range) { + PAIMON_ASSIGN_OR_RAISE(auto split, ScanData(table_path, partition_filters)); + PAIMON_ASSIGN_OR_RAISE(auto index_commit_msg, RowRangeGlobalIndexWriter::WriteIndex( + table_path, index_field_name, index_type, + std::make_shared( + split, std::vector({range})), + options, pool_)); + return Commit(table_path, {index_commit_msg}); } - Status ScanAndRead(const std::string& table_path, const std::vector& read_schema, - const std::shared_ptr& expected_array, - const std::shared_ptr& predicate, - const std::vector& row_ranges, - const std::map& id_to_score) const { - // scan + Result> ScanGlobalIndexAndData( + const std::string& table_path, const std::shared_ptr& predicate, + const std::map& options = {}, + const std::shared_ptr& index_result = nullptr) const { ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.SetPredicate(predicate).SetRowRanges(row_ranges); + scan_context_builder.SetPredicate(predicate).SetOptions(options).SetGlobalIndexResult( + index_result); PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); - if (!expected_array) { - if (!result_plan->Splits().empty()) { - return Status::Invalid("check_scan_plan_when_empty_result but plan is not empty"); + return result_plan; + } + + Result> ScanDataWithIndexResult( + const std::string& table_path, const std::shared_ptr& predicate, + const std::vector& row_ranges, const std::map& id_to_score) const { + std::shared_ptr index_result; + if (id_to_score.empty()) { + index_result = BitmapGlobalIndexResult::FromRanges(row_ranges); + } else { + RoaringBitmap64 bitmap; + for (const auto& range : row_ranges) { + bitmap.AddRange(range.from, range.to + 1); } + std::vector scores; + for (auto iter = bitmap.Begin(); iter != bitmap.End(); ++iter) { + scores.push_back(id_to_score.at(*iter)); + } + index_result = + std::make_shared(std::move(bitmap), std::move(scores)); } + return ScanGlobalIndexAndData(table_path, predicate, /*options=*/{}, index_result); + } - // read + Status ReadData(const std::string& table_path, const std::vector& read_schema, + const std::shared_ptr& expected_array, + const std::shared_ptr& predicate, + const std::shared_ptr& result_plan) const { auto splits = result_plan->Splits(); ReadContextBuilder read_context_builder(table_path); read_context_builder.SetReadSchema(read_schema).SetPredicate(predicate); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); - PAIMON_ASSIGN_OR_RAISE(auto read_splits, CreateReadSplit(splits, row_ranges, id_to_score)); - PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(read_splits)); + PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); PAIMON_ASSIGN_OR_RAISE(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); @@ -770,13 +766,8 @@ TEST_P(GlobalIndexTest, TestWriteCommitScanReadIndex) { ASSERT_OK_AND_ASSIGN(auto commit_msgs, WriteArray(table_path, write_cols, src_array)); ASSERT_OK(Commit(table_path, commit_msgs)); - ASSERT_OK_AND_ASSIGN(auto split, ScanData(table_path, /*partition_filters=*/{})); - ASSERT_OK_AND_ASSIGN(auto index_commit_msg, RowRangeGlobalIndexWriter::WriteIndex( - table_path, "f0", "bitmap", - std::make_shared( - split, std::vector({Range(0, 7)})), - /*options=*/{}, pool_)); - ASSERT_OK(Commit(table_path, {index_commit_msg})); + ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{}, "f0", "bitmap", + /*options=*/{}, Range(0, 7))); ASSERT_OK_AND_ASSIGN(auto global_index_scan, GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, @@ -819,23 +810,12 @@ TEST_P(GlobalIndexTest, TestWriteCommitScanReadIndexWithPartition) { WriteArray(table_path, partition, write_cols, src_array)); ASSERT_OK(Commit(table_path, commit_msgs)); - ASSERT_OK_AND_ASSIGN(auto split, ScanData(table_path, /*partition_filters=*/{partition})); // write bitmap index - ASSERT_OK_AND_ASSIGN( - auto bitmap_commit_msg, - RowRangeGlobalIndexWriter::WriteIndex( - table_path, "f0", "bitmap", - std::make_shared(split, std::vector({expected_range})), - /*options=*/{}, pool_)); - ASSERT_OK(Commit(table_path, {bitmap_commit_msg})); + ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{partition}, "f0", "bitmap", + /*options=*/{}, expected_range)); // write and commit lumina index - ASSERT_OK_AND_ASSIGN( - auto lumina_commit_msg, - RowRangeGlobalIndexWriter::WriteIndex( - table_path, "f1", "lumina", - std::make_shared(split, std::vector({expected_range})), - lumina_options, pool_)); - ASSERT_OK(Commit(table_path, {lumina_commit_msg})); + ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{partition}, "f1", "lumina", + /*options=*/lumina_options, expected_range)); }; // write partition f2 = 10 @@ -861,52 +841,56 @@ TEST_P(GlobalIndexTest, TestWriteCommitScanReadIndexWithPartition) { .ValueOrDie()); write_data_and_index(src_array2, {{"f2", "20"}}, Range(4, 8)); - auto scan_and_check_result = - [&](const std::map& partition, const Range& expected_range, - GlobalIndexReader::TopKPreFilter filter, int32_t k, const std::string& bitmap_result, - const std::string& lumina_result, const std::vector& read_row_ranges, - const std::shared_ptr& expected_array, - const std::map& id_to_score) { - std::vector> partitions = {partition}; - ASSERT_OK_AND_ASSIGN(auto global_index_scan, - GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, - partitions, lumina_options, - /*file_system=*/nullptr, pool_)); - ASSERT_OK_AND_ASSIGN(std::vector ranges, global_index_scan->GetRowRangeList()); - ASSERT_EQ(ranges, std::vector({expected_range})); + auto scan_and_check_result = [&](const std::map& partition, + const Range& expected_range, + GlobalIndexReader::TopKPreFilter filter, int32_t k, + const std::string& bitmap_result, + const std::string& lumina_result, + const std::vector& read_row_ranges, + const std::shared_ptr& expected_array, + const std::map& id_to_score) { + std::vector> partitions = {partition}; + ASSERT_OK_AND_ASSIGN(auto global_index_scan, + GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, + partitions, lumina_options, + /*file_system=*/nullptr, pool_)); + ASSERT_OK_AND_ASSIGN(std::vector ranges, global_index_scan->GetRowRangeList()); + ASSERT_EQ(ranges, std::vector({expected_range})); - ASSERT_OK_AND_ASSIGN(auto range_scanner, - global_index_scan->CreateRangeScan(expected_range)); + ASSERT_OK_AND_ASSIGN(auto range_scanner, + global_index_scan->CreateRangeScan(expected_range)); - // check bitmap index - ASSERT_OK_AND_ASSIGN(auto evaluator, range_scanner->CreateIndexEvaluator()); + // check bitmap index + ASSERT_OK_AND_ASSIGN(auto evaluator, range_scanner->CreateIndexEvaluator()); - auto predicate1 = - PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, - Literal(FieldType::STRING, "Alice", 5)); - auto predicate2 = - PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, - Literal(FieldType::STRING, "Paul", 4)); - ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::Or({predicate1, predicate2})); - - ASSERT_OK_AND_ASSIGN(auto index_result, evaluator->Evaluate(predicate)); - ASSERT_TRUE(index_result); - ASSERT_EQ(index_result.value()->ToString(), bitmap_result); - - // check lumina index - ASSERT_OK_AND_ASSIGN(auto lumina_reader, range_scanner->CreateReader("f1", "lumina")); - - std::vector query = {1.0f, 1.0f, 1.0f, 1.1f}; - ASSERT_OK_AND_ASSIGN(auto topk_result, lumina_reader->VisitTopK(k, query, filter, - /*predicate*/ nullptr)); - ASSERT_EQ(topk_result->ToString(), lumina_result); - - // check read array - std::vector read_field_names = schema->field_names(); - read_field_names.push_back("_INDEX_SCORE"); - ASSERT_OK(ScanAndRead(table_path, read_field_names, expected_array, - /*predicate=*/nullptr, read_row_ranges, id_to_score)); - }; + auto predicate1 = + PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, + Literal(FieldType::STRING, "Alice", 5)); + auto predicate2 = + PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, + Literal(FieldType::STRING, "Paul", 4)); + ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::Or({predicate1, predicate2})); + + ASSERT_OK_AND_ASSIGN(auto index_result, evaluator->Evaluate(predicate)); + ASSERT_TRUE(index_result); + ASSERT_EQ(index_result.value()->ToString(), bitmap_result); + + // check lumina index + ASSERT_OK_AND_ASSIGN(auto lumina_reader, range_scanner->CreateReader("f1", "lumina")); + + std::vector query = {1.0f, 1.0f, 1.0f, 1.1f}; + ASSERT_OK_AND_ASSIGN(auto topk_result, lumina_reader->VisitTopK(k, query, filter, + /*predicate*/ nullptr)); + ASSERT_EQ(topk_result->ToString(), lumina_result); + + // check read array + std::vector read_field_names = schema->field_names(); + read_field_names.push_back("_INDEX_SCORE"); + ASSERT_OK_AND_ASSIGN(auto plan, ScanGlobalIndexAndData(table_path, /*predicate=*/nullptr, + /*options=*/{}, topk_result)); + ASSERT_OK(ReadData(table_path, read_field_names, expected_array, + /*predicate=*/nullptr, plan)); + }; auto result_fields = fields; result_fields.insert(result_fields.begin(), SpecialFields::ValueKind().ArrowField()); @@ -986,15 +970,10 @@ TEST_P(GlobalIndexTest, TestWriteCommitScanReadIndexWithScore) { ASSERT_OK_AND_ASSIGN(auto commit_msgs, WriteArray(table_path, write_cols, src_array)); ASSERT_OK(Commit(table_path, commit_msgs)); - ASSERT_OK_AND_ASSIGN(auto split, ScanData(table_path, /*partition_filters=*/{})); // write and commit lumina index - ASSERT_OK_AND_ASSIGN(auto lumina_commit_msg, RowRangeGlobalIndexWriter::WriteIndex( - table_path, "f1", "lumina", - std::make_shared( - split, std::vector({Range(0, 8)})), - lumina_options, pool_)); - ASSERT_OK(Commit(table_path, {lumina_commit_msg})); + ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{}, "f1", "lumina", + /*options=*/lumina_options, Range(0, 8))); auto scan_and_check_result = [&](const std::vector& read_row_ranges, const std::shared_ptr& expected_array, @@ -1002,8 +981,10 @@ TEST_P(GlobalIndexTest, TestWriteCommitScanReadIndexWithScore) { // check read array std::vector read_field_names = schema->field_names(); read_field_names.push_back("_INDEX_SCORE"); - ASSERT_OK(ScanAndRead(table_path, read_field_names, expected_array, - /*predicate=*/nullptr, read_row_ranges, id_to_score)); + ASSERT_OK_AND_ASSIGN(auto plan, ScanDataWithIndexResult(table_path, /*predicate=*/nullptr, + read_row_ranges, id_to_score)); + ASSERT_OK(ReadData(table_path, read_field_names, expected_array, + /*predicate=*/nullptr, plan)); }; auto result_fields = fields; @@ -1060,6 +1041,566 @@ TEST_P(GlobalIndexTest, TestWriteCommitScanReadIndexWithScore) { } } +TEST_P(GlobalIndexTest, TestDataEvolutionBatchScan) { + CreateTable(); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields_); + // write and commit data + std::vector write_cols = schema->field_names(); + auto src_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_), R"([ +["Alice", 10, 1, 11.1], +["Bob", 10, 1, 12.1], +["Emily", 10, 0, 13.1], +["Tony", 10, 0, 14.1], +["Lucy", 20, 1, 15.1], +["Bob", 10, 1, 16.1], +["Tony", 20, 0, 17.1], +["Alice", 20, null, 18.1] + ])") + .ValueOrDie()); + + ASSERT_OK_AND_ASSIGN(auto commit_msgs, WriteArray(table_path, write_cols, src_array)); + ASSERT_OK(Commit(table_path, commit_msgs)); + + auto result_fields = fields_; + result_fields.insert(result_fields.begin(), SpecialFields::ValueKind().ArrowField()); + auto expected_all_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(result_fields), R"([ +[0, "Alice", 10, 1, 11.1], +[0, "Bob", 10, 1, 12.1], +[0, "Emily", 10, 0, 13.1], +[0, "Tony", 10, 0, 14.1], +[0, "Lucy", 20, 1, 15.1], +[0, "Bob", 10, 1, 16.1], +[0, "Tony", 20, 0, 17.1], +[0, "Alice", 20, null, 18.1] + ])") + .ValueOrDie()); + + { + // read when no index is built + auto predicate = + PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, + Literal(FieldType::STRING, "Alice", 5)); + ASSERT_OK_AND_ASSIGN(auto plan, ScanGlobalIndexAndData(table_path, predicate)); + ASSERT_OK(ReadData(table_path, write_cols, expected_all_array, predicate, plan)); + } + + // write and commit global index + ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{}, "f0", "bitmap", /*options=*/{}, + Range(0, 7))); + + // scan and read with global index + { + auto predicate = + PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, + Literal(FieldType::STRING, "Alice", 5)); + ASSERT_OK_AND_ASSIGN(auto plan, ScanGlobalIndexAndData(table_path, predicate)); + + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(result_fields), R"([ +[0, "Alice", 10, 1, 11.1], +[0, "Alice", 20, null, 18.1] + ])") + .ValueOrDie()); + ASSERT_OK(ReadData(table_path, write_cols, expected_array, predicate, plan)); + } + { + auto predicate = + PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, + Literal(FieldType::STRING, "Alice2", 6)); + ASSERT_OK_AND_ASSIGN(auto plan, ScanGlobalIndexAndData(table_path, predicate)); + + ASSERT_OK(ReadData(table_path, write_cols, /*expected_array=*/nullptr, predicate, plan)); + } + { + auto predicate1 = + PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, + Literal(FieldType::STRING, "Alice", 5)); + auto predicate2 = + PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, + Literal(FieldType::STRING, "Bob", 3)); + ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::Or({predicate1, predicate2})); + ASSERT_OK_AND_ASSIGN(auto plan, ScanGlobalIndexAndData(table_path, predicate)); + + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(result_fields), R"([ +[0, "Alice", 10, 1, 11.1], +[0, "Bob", 10, 1, 12.1], +[0, "Bob", 10, 1, 16.1], +[0, "Alice", 20, null, 18.1] + ])") + .ValueOrDie()); + ASSERT_OK(ReadData(table_path, write_cols, expected_array, predicate, plan)); + } + { + ASSERT_OK_AND_ASSIGN(auto plan, ScanGlobalIndexAndData(table_path, /*predicate=*/nullptr)); + ASSERT_OK( + ReadData(table_path, write_cols, expected_all_array, /*predicate=*/nullptr, plan)); + } + { + // f1 does not have global index + auto predicate = PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", + FieldType::INT, Literal(19)); + ASSERT_OK_AND_ASSIGN(auto plan, ScanGlobalIndexAndData(table_path, predicate)); + ASSERT_OK(ReadData(table_path, write_cols, expected_all_array, predicate, plan)); + } + { + // disable global index + auto predicate = + PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, + Literal(FieldType::STRING, "Alice", 5)); + ASSERT_OK_AND_ASSIGN( + auto plan, + ScanGlobalIndexAndData(table_path, predicate, {{"global-index.enabled", "false"}})); + ASSERT_OK(ReadData(table_path, write_cols, expected_all_array, predicate, plan)); + } +} + +TEST_P(GlobalIndexTest, TestDataEvolutionBatchScanWithOnlyOnePartitionHasIndex) { + CreateTable(/*partition_keys=*/{"f1"}); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields_); + // write and commit data + std::vector write_cols = schema->field_names(); + auto src_array1 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_), R"([ +["Alice", 10, 1, 11.1], +["Bob", 10, 1, 12.1], +["Emily", 10, 0, 13.1], +["Tony", 10, 0, 14.1], +["Bob", 10, 1, 16.1] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, + WriteArray(table_path, {{"f1", "10"}}, write_cols, src_array1)); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + auto src_array2 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_), R"([ +["Lucy", 20, 1, 15.1], +["Tony", 20, 0, 17.1], +["Alice", 20, null, 18.1] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs2, + WriteArray(table_path, {{"f1", "20"}}, write_cols, src_array2)); + ASSERT_OK(Commit(table_path, commit_msgs2)); + + // write and commit global index for f1 = 10 + ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{{{"f1", "10"}}}, "f0", "bitmap", + /*options=*/{}, Range(0, 4))); + + auto result_fields = fields_; + result_fields.insert(result_fields.begin(), SpecialFields::ValueKind().ArrowField()); + { + // only f1 = 10 partition has index, f1 = 20 partition will not be filtered + auto predicate = + PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, + Literal(FieldType::STRING, "Alice", 5)); + ASSERT_OK_AND_ASSIGN(auto plan, ScanGlobalIndexAndData(table_path, predicate)); + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(result_fields), R"([ +[0, "Alice", 10, 1, 11.1], +[0, "Lucy", 20, 1, 15.1], +[0, "Tony", 20, 0, 17.1], +[0, "Alice", 20, null, 18.1] + ])") + .ValueOrDie()); + + ASSERT_OK(ReadData(table_path, write_cols, expected_array, predicate, plan)); + } +} + +TEST_P(GlobalIndexTest, TestDataEvolutionBatchScanWithTwoIndexInDiffTwoPartition) { + CreateTable(/*partition_keys=*/{"f1"}); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields_); + // write and commit data + std::vector write_cols = schema->field_names(); + auto src_array1 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_), R"([ +["Alice", 10, 1, 11.1], +["Bob", 10, 1, 12.1], +["Emily", 10, 0, 13.1], +["Tony", 10, 0, 14.1], +["Bob", 10, 1, 16.1] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, + WriteArray(table_path, {{"f1", "10"}}, write_cols, src_array1)); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + auto src_array2 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_), R"([ +["Lucy", 20, 1, 15.1], +["Tony", 20, 0, 17.1], +["Alice", 20, null, 18.1] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs2, + WriteArray(table_path, {{"f1", "20"}}, write_cols, src_array2)); + ASSERT_OK(Commit(table_path, commit_msgs2)); + + // write and commit global index for f1 = 10 + ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{{{"f1", "10"}}}, "f0", "bitmap", + /*options=*/{}, Range(0, 4))); + + // write and commit global index for f1 = 20 + ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{{{"f1", "20"}}}, "f2", "bitmap", + /*options=*/{}, Range(5, 7))); + + auto result_fields = fields_; + result_fields.insert(result_fields.begin(), SpecialFields::ValueKind().ArrowField()); + { + // only f1 = 10 partition has f0 index, f1 = 20 partition will not be filtered + auto predicate = + PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, + Literal(FieldType::STRING, "Alice", 5)); + ASSERT_OK_AND_ASSIGN(auto plan, ScanGlobalIndexAndData(table_path, predicate)); + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(result_fields), R"([ +[0, "Alice", 10, 1, 11.1], +[0, "Lucy", 20, 1, 15.1], +[0, "Tony", 20, 0, 17.1], +[0, "Alice", 20, null, 18.1] + ])") + .ValueOrDie()); + + ASSERT_OK(ReadData(table_path, write_cols, expected_array, predicate, plan)); + } + { + // only f1 = 20 partition has f2 index, f1 = 10 partition will not be filtered + auto predicate = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", + FieldType::INT, Literal(1)); + ASSERT_OK_AND_ASSIGN(auto plan, ScanGlobalIndexAndData(table_path, predicate)); + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(result_fields), R"([ +[0, "Alice", 10, 1, 11.1], +[0, "Bob", 10, 1, 12.1], +[0, "Emily", 10, 0, 13.1], +[0, "Tony", 10, 0, 14.1], +[0, "Bob", 10, 1, 16.1], +[0, "Lucy", 20, 1, 15.1] + ])") + .ValueOrDie()); + + ASSERT_OK(ReadData(table_path, write_cols, expected_array, predicate, plan)); + } + { + auto predicate1 = + PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, + Literal(FieldType::STRING, "Alice", 5)); + auto predicate2 = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", + FieldType::INT, Literal(1)); + ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({predicate1, predicate2})); + ASSERT_OK_AND_ASSIGN(auto plan, ScanGlobalIndexAndData(table_path, predicate)); + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(result_fields), R"([ +[0, "Alice", 10, 1, 11.1], +[0, "Lucy", 20, 1, 15.1] + ])") + .ValueOrDie()); + + ASSERT_OK(ReadData(table_path, write_cols, expected_array, predicate, plan)); + } + { + // predicate2 is partition filter + auto predicate1 = + PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, + Literal(FieldType::STRING, "Alice", 5)); + auto predicate2 = PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", + FieldType::INT, Literal(10)); + ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({predicate1, predicate2})); + ASSERT_OK_AND_ASSIGN(auto plan, ScanGlobalIndexAndData(table_path, predicate)); + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(result_fields), R"([ +[0, "Alice", 10, 1, 11.1] + ])") + .ValueOrDie()); + + ASSERT_OK(ReadData(table_path, write_cols, expected_array, predicate, plan)); + } +} + +TEST_P(GlobalIndexTest, TestDataEvolutionBatchScanWithTwoPartitionAllWithIndex) { + CreateTable(/*partition_keys=*/{"f1"}); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields_); + // write and commit data + std::vector write_cols = schema->field_names(); + auto src_array1 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_), R"([ +["Alice", 10, 1, 11.1], +["Bob", 10, 1, 12.1], +["Emily", 10, 0, 13.1], +["Tony", 10, 0, 14.1], +["Bob", 10, 1, 16.1] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, + WriteArray(table_path, {{"f1", "10"}}, write_cols, src_array1)); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + auto src_array2 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_), R"([ +["Lucy", 20, 1, 15.1], +["Tony", 20, 0, 17.1], +["Alice", 20, null, 18.1] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs2, + WriteArray(table_path, {{"f1", "20"}}, write_cols, src_array2)); + ASSERT_OK(Commit(table_path, commit_msgs2)); + + // write and commit global index for f1 = 10 + ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{{{"f1", "10"}}}, "f0", "bitmap", + /*options=*/{}, Range(0, 4))); + + // write and commit global index for f1 = 20 + ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{{{"f1", "20"}}}, "f0", "bitmap", + /*options=*/{}, Range(5, 7))); + + auto result_fields = fields_; + result_fields.insert(result_fields.begin(), SpecialFields::ValueKind().ArrowField()); + { + auto predicate = + PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, + Literal(FieldType::STRING, "Alice", 5)); + ASSERT_OK_AND_ASSIGN(auto plan, ScanGlobalIndexAndData(table_path, predicate)); + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(result_fields), R"([ +[0, "Alice", 10, 1, 11.1], +[0, "Alice", 20, null, 18.1] + ])") + .ValueOrDie()); + + ASSERT_OK(ReadData(table_path, write_cols, expected_array, predicate, plan)); + } + { + // predicate2 is partition filter + auto predicate1 = + PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, + Literal(FieldType::STRING, "Alice", 5)); + auto predicate2 = PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", + FieldType::INT, Literal(10)); + ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({predicate1, predicate2})); + ASSERT_OK_AND_ASSIGN(auto plan, ScanGlobalIndexAndData(table_path, predicate)); + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(result_fields), R"([ +[0, "Alice", 10, 1, 11.1] + ])") + .ValueOrDie()); + + ASSERT_OK(ReadData(table_path, write_cols, expected_array, predicate, plan)); + } + { + // predicate2 is partition filter + auto predicate1 = + PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, + Literal(FieldType::STRING, "Alice", 5)); + auto predicate2 = PredicateBuilder::LessThan(/*field_index=*/1, /*field_name=*/"f1", + FieldType::INT, Literal(20)); + ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({predicate1, predicate2})); + ASSERT_OK_AND_ASSIGN(auto plan, ScanGlobalIndexAndData(table_path, predicate)); + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(result_fields), R"([ +[0, "Alice", 10, 1, 11.1] + ])") + .ValueOrDie()); + + ASSERT_OK(ReadData(table_path, write_cols, expected_array, predicate, plan)); + } + { + // predicate2 is partition filter + auto predicate1 = + PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, + Literal(FieldType::STRING, "Alice", 5)); + auto predicate2 = PredicateBuilder::LessThan(/*field_index=*/1, /*field_name=*/"f1", + FieldType::INT, Literal(30)); + ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({predicate1, predicate2})); + ASSERT_OK_AND_ASSIGN(auto plan, ScanGlobalIndexAndData(table_path, predicate)); + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(result_fields), R"([ +[0, "Alice", 10, 1, 11.1], +[0, "Alice", 20, null, 18.1] + ])") + .ValueOrDie()); + + ASSERT_OK(ReadData(table_path, write_cols, expected_array, predicate, plan)); + } +} + +TEST_P(GlobalIndexTest, TestInvalidGetRowRangeListWithIndexRangeMismatchViaDifferentType) { + arrow::FieldVector fields = { + arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::list(arrow::float32())), + arrow::field("f2", arrow::int32()), arrow::field("f3", arrow::float64())}; + std::map lumina_options = { + {"lumina.dimension", "4"}, + {"lumina.indextype", "bruteforce"}, + {"lumina.distance.metric", "l2"}, + {"lumina.encoding.type", "encoding.rawf32"}, + {"lumina.search.threadcount", "10"}}; + auto schema = arrow::schema(fields); + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(/*partition_keys=*/{"f2"}, schema, options); + + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + std::vector write_cols = schema->field_names(); + // write partition f2 = 10 + auto src_array1 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ +["Alice", [0.0, 0.0, 0.0, 0.0], 10, 11.1], +["Bob", [0.0, 1.0, 0.0, 1.0], 10, 12.1], +["Emily", [1.0, 0.0, 1.0, 0.0], 10, 13.1], +["Tony", [1.0, 1.0, 1.0, 1.0], 10, 14.1] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, + WriteArray(table_path, {{"f2", "10"}}, write_cols, src_array1)); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + // write partition f2 = 20 + auto src_array2 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ +["Lucy", [10.0, 10.0, 10.0, 10.0], 20, 15.1], +["Bob", [10.0, 11.0, 10.0, 11.0], 20, 16.1], +["Tony", [11.0, 10.0, 11.0, 10.0], 20, 17.1], +["Alice", [11.0, 11.0, 11.0, 11.0], 20, 18.1], +["Paul", [10.0, 10.0, 10.0, 10.0], 20, 19.1] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs2, + WriteArray(table_path, {{"f2", "20"}}, write_cols, src_array2)); + ASSERT_OK(Commit(table_path, commit_msgs2)); + + // write and commit bitmap global index for f2 = 10 + ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{{{"f2", "10"}}}, "f0", "bitmap", + /*options=*/{}, Range(0, 3))); + + // write and commit lumina global index for f2 = 20 + ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{{{"f2", "20"}}}, "f1", "lumina", + /*options=*/lumina_options, Range(4, 8))); + + ASSERT_OK_AND_ASSIGN( + auto global_index_scan, + GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, + /*partitions=*/std::nullopt, /*options=*/lumina_options, + /*file_system=*/nullptr, pool_)); + ASSERT_NOK_WITH_MSG(global_index_scan->GetRowRangeList(), + "Inconsistent row ranges among index types"); +} + +TEST_P(GlobalIndexTest, TestDataEvolutionBatchScanWithPartitionWithTwoFields) { + CreateTable(/*partition_keys=*/{"f1", "f2"}); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields_); + // write and commit data + std::vector write_cols = schema->field_names(); + + auto src_array1 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_), R"([ +["Alice", 10, 1, 11.1], +["Bob", 10, 1, 12.1], +["Bob", 10, 1, 16.1] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {{"f1", "10"}, {"f2", "1"}}, + write_cols, src_array1)); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + auto src_array2 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_), R"([ +["Lucy", 20, 1, 15.1] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs2, WriteArray(table_path, {{"f1", "20"}, {"f2", "1"}}, + write_cols, src_array2)); + ASSERT_OK(Commit(table_path, commit_msgs2)); + + auto src_array3 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_), R"([ +["Emily", 10, 0, 13.1], +["Tony", 10, 0, 14.1] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs3, WriteArray(table_path, {{"f1", "10"}, {"f2", "0"}}, + write_cols, src_array3)); + ASSERT_OK(Commit(table_path, commit_msgs3)); + + // build index + ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{{{"f1", "10"}, {"f2", "1"}}}, "f0", + "bitmap", + /*options=*/{}, Range(0, 2))); + + ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{{{"f1", "20"}, {"f2", "1"}}}, "f0", + "bitmap", + /*options=*/{}, Range(3, 3))); + + ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{{{"f1", "10"}, {"f2", "0"}}}, "f0", + "bitmap", + /*options=*/{}, Range(4, 5))); + + auto result_fields = fields_; + result_fields.insert(result_fields.begin(), SpecialFields::ValueKind().ArrowField()); + { + auto predicate1 = PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", + FieldType::INT, Literal(10)); + auto predicate2 = + PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, + Literal(FieldType::STRING, "Bob", 3)); + ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({predicate1, predicate2})); + + ASSERT_OK_AND_ASSIGN(auto plan, ScanGlobalIndexAndData(table_path, predicate)); + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(result_fields), R"([ +[0, "Bob", 10, 1, 12.1], +[0, "Bob", 10, 1, 16.1] + ])") + .ValueOrDie()); + ASSERT_OK(ReadData(table_path, write_cols, expected_array, predicate, plan)); + } + { + auto predicate1 = PredicateBuilder::GreaterThan( + /*field_index=*/1, /*field_name=*/"f1", FieldType::INT, Literal(10)); + auto predicate2 = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", + FieldType::INT, Literal(1)); + ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::Or({predicate1, predicate2})); + + ASSERT_OK_AND_ASSIGN(auto plan, ScanGlobalIndexAndData(table_path, predicate)); + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(result_fields), R"([ +[0, "Alice", 10, 1, 11.1], +[0, "Bob", 10, 1, 12.1], +[0, "Bob", 10, 1, 16.1], +[0, "Lucy", 20, 1, 15.1] + ])") + .ValueOrDie()); + ASSERT_OK(ReadData(table_path, write_cols, expected_array, predicate, plan)); + } + { + auto predicate1 = PredicateBuilder::GreaterThan( + /*field_index=*/1, /*field_name=*/"f1", FieldType::INT, Literal(10)); + auto predicate2 = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", + FieldType::INT, Literal(1)); + ASSERT_OK_AND_ASSIGN(auto or_predicate, PredicateBuilder::Or({predicate1, predicate2})); + + auto predicate3 = + PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, + Literal(FieldType::STRING, "Emily", 5)); + ASSERT_OK_AND_ASSIGN(auto and_predicate, PredicateBuilder::And({predicate3, or_predicate})); + + ASSERT_OK_AND_ASSIGN(auto plan, ScanGlobalIndexAndData(table_path, and_predicate)); + ASSERT_OK( + ReadData(table_path, write_cols, /*expected_array=*/nullptr, and_predicate, plan)); + } +} + std::vector GetTestValuesForGlobalIndexTest() { std::vector values = {"parquet"}; #ifdef PAIMON_ENABLE_ORC From 691e00b68962e70137001bddcf77e30245996277 Mon Sep 17 00:00:00 2001 From: "lisizhuo.lsz" Date: Mon, 22 Dec 2025 03:46:45 +0000 Subject: [PATCH 2/2] fix --- .../core/global_index/indexed_split_impl.h | 2 +- .../core/global_index/indexed_split_test.cpp | 19 +++++++++++++++++++ src/paimon/core/schema/table_schema.cpp | 1 + .../source/data_evolution_batch_scan.cpp | 13 +++++++++---- test/inte/global_index_test.cpp | 4 ++-- 5 files changed, 32 insertions(+), 7 deletions(-) diff --git a/src/paimon/core/global_index/indexed_split_impl.h b/src/paimon/core/global_index/indexed_split_impl.h index 7f1cad75e..c6b407ce6 100644 --- a/src/paimon/core/global_index/indexed_split_impl.h +++ b/src/paimon/core/global_index/indexed_split_impl.h @@ -91,7 +91,7 @@ class IndexedSplitImpl : public IndexedSplit { Status Validate() const { if ((row_ranges_.empty() && !data_split_->DataFiles().empty()) || (!row_ranges_.empty() && data_split_->DataFiles().empty())) { - return Status::Invalid("invalid IndexedSplit: row ranges mismatch data files"); + return Status::Invalid("Invalid IndexedSplit: row ranges mismatch data files."); } if (!scores_.empty()) { size_t row_count = 0; diff --git a/src/paimon/core/global_index/indexed_split_test.cpp b/src/paimon/core/global_index/indexed_split_test.cpp index b5dae50d8..30e9374c6 100644 --- a/src/paimon/core/global_index/indexed_split_test.cpp +++ b/src/paimon/core/global_index/indexed_split_test.cpp @@ -169,5 +169,24 @@ TEST(IndexedSplitTest, TestValidate) { ASSERT_NOK_WITH_MSG(split.Validate(), "Scores length does not match row ranges in indexed split."); } + { + std::vector row_ranges = {}; + IndexedSplitImpl split(data_split, row_ranges); + ASSERT_NOK_WITH_MSG(split.Validate(), + "Invalid IndexedSplit: row ranges mismatch data files."); + } + { + std::vector row_ranges = {Range(10, 12)}; + DataSplitImpl::Builder empty_builder( + /*partition=*/BinaryRow::EmptyRow(), + /*bucket=*/0, /*bucket_path=*/ + "data/test_table/bucket-0", std::vector>({})); + auto empty_data_split = std::dynamic_pointer_cast( + empty_builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build().value()); + + IndexedSplitImpl split(empty_data_split, row_ranges); + ASSERT_NOK_WITH_MSG(split.Validate(), + "Invalid IndexedSplit: row ranges mismatch data files."); + } } } // namespace paimon::test diff --git a/src/paimon/core/schema/table_schema.cpp b/src/paimon/core/schema/table_schema.cpp index e5ee56d3f..96cfecf07 100644 --- a/src/paimon/core/schema/table_schema.cpp +++ b/src/paimon/core/schema/table_schema.cpp @@ -200,6 +200,7 @@ Result TableSchema::GetField(int32_t field_id) const { Result> TableSchema::GetFields( const std::vector& field_names) const { std::vector data_fields; + data_fields.reserve(field_names.size()); for (const auto& name : field_names) { PAIMON_ASSIGN_OR_RAISE(DataField field, GetField(name)); data_fields.emplace_back(field); diff --git a/src/paimon/core/table/source/data_evolution_batch_scan.cpp b/src/paimon/core/table/source/data_evolution_batch_scan.cpp index 1a8b59078..8e1377f60 100644 --- a/src/paimon/core/table/source/data_evolution_batch_scan.cpp +++ b/src/paimon/core/table/source/data_evolution_batch_scan.cpp @@ -37,14 +37,16 @@ DataEvolutionBatchScan::DataEvolutionBatchScan( Result> DataEvolutionBatchScan::CreatePlan() { std::optional> row_ranges; - if (!global_index_result_) { + std::shared_ptr final_global_index_result = global_index_result_; + if (!final_global_index_result) { PAIMON_ASSIGN_OR_RAISE(std::optional> index_result, EvalGlobalIndex()); if (index_result) { + final_global_index_result = index_result.value(); PAIMON_ASSIGN_OR_RAISE(row_ranges, index_result.value()->ToRanges()); } } else { - PAIMON_ASSIGN_OR_RAISE(row_ranges, global_index_result_->ToRanges()); + PAIMON_ASSIGN_OR_RAISE(row_ranges, final_global_index_result->ToRanges()); } if (!row_ranges) { return batch_scan_->CreatePlan(); @@ -52,7 +54,8 @@ Result> DataEvolutionBatchScan::CreatePlan() { batch_scan_->WithRowRanges(row_ranges.value()); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_plan, batch_scan_->CreatePlan()); std::map id_to_score; - if (auto topk_result = std::dynamic_pointer_cast(global_index_result_)) { + if (auto topk_result = + std::dynamic_pointer_cast(final_global_index_result)) { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr topk_iter, topk_result->CreateTopKIterator()); while (topk_iter->HasNext()) { @@ -68,8 +71,10 @@ Result> DataEvolutionBatchScan::WrapToIndexedSplits( const std::map& id_to_score) const { std::vector sorted_row_ranges = Range::SortAndMergeOverlap(row_ranges, /*adjacent=*/true); + auto data_splits = data_plan->Splits(); std::vector> indexed_splits; - for (const auto& split : data_plan->Splits()) { + indexed_splits.reserve(data_splits.size()); + for (const auto& split : data_splits) { auto data_split = std::dynamic_pointer_cast(split); if (!data_split) { return Status::Invalid("Cannot cast split to DataSplit when create IndexedSplit"); diff --git a/test/inte/global_index_test.cpp b/test/inte/global_index_test.cpp index d16341fb9..dc3db76ab 100644 --- a/test/inte/global_index_test.cpp +++ b/test/inte/global_index_test.cpp @@ -1505,7 +1505,7 @@ TEST_P(GlobalIndexTest, TestDataEvolutionBatchScanWithPartitionWithTwoFields) { auto src_array1 = std::dynamic_pointer_cast( arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_), R"([ -["Alice", 10, 1, 11.1], +["Alice", 10, 1, 11.1], ["Bob", 10, 1, 12.1], ["Bob", 10, 1, 16.1] ])") @@ -1575,7 +1575,7 @@ TEST_P(GlobalIndexTest, TestDataEvolutionBatchScanWithPartitionWithTwoFields) { ASSERT_OK_AND_ASSIGN(auto plan, ScanGlobalIndexAndData(table_path, predicate)); auto expected_array = std::dynamic_pointer_cast( arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(result_fields), R"([ -[0, "Alice", 10, 1, 11.1], +[0, "Alice", 10, 1, 11.1], [0, "Bob", 10, 1, 12.1], [0, "Bob", 10, 1, 16.1], [0, "Lucy", 20, 1, 15.1]