From 585236e2d7e7a474ae925469438c1bf7258dccbc Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Wed, 20 May 2026 20:46:02 +0100 Subject: [PATCH] Upgrade DF to 54 Signed-off-by: Adam Gutglick --- Cargo.lock | 1471 ++++++++++++------- Cargo.toml | 64 +- benchmarks/datafusion-bench/src/lib.rs | 4 +- benchmarks/datafusion-bench/src/main.rs | 9 +- vortex-datafusion/src/convert/exprs.rs | 63 +- vortex-datafusion/src/persistent/format.rs | 10 - vortex-datafusion/src/persistent/metrics.rs | 11 +- vortex-datafusion/src/persistent/opener.rs | 27 +- vortex-datafusion/src/persistent/sink.rs | 5 - vortex-datafusion/src/persistent/source.rs | 5 - vortex-datafusion/src/v2/source.rs | 11 +- vortex-datafusion/src/v2/table.rs | 5 - 12 files changed, 1010 insertions(+), 675 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1229e176999..31946ffca3c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -145,35 +145,6 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" -[[package]] -name = "apache-avro" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf" -dependencies = [ - "bigdecimal", - "bon", - "bzip2", - "crc32fast", - "digest 0.10.7", - "liblzma", - "log", - "miniz_oxide", - "num-bigint", - "quad-rand", - "rand 0.9.4", - "regex-lite", - "serde", - "serde_bytes", - "serde_json", - "snap", - "strum 0.27.2", - "strum_macros 0.27.2", - "thiserror 2.0.18", - "uuid", - "zstd", -] - [[package]] name = "approx" version = "0.5.1" @@ -282,6 +253,30 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arrow-avro" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "049230728cd6e093088c8d231b4beede184e35cad7777c1505c0d5a8571f4376" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", + "bytes", + "bzip2", + "crc", + "flate2", + "indexmap 2.14.0", + "liblzma", + "rand 0.9.4", + "serde", + "serde_json", + "snap", + "strum_macros 0.28.0", + "uuid", + "zstd", +] + [[package]] name = "arrow-buffer" version = "58.3.0" @@ -787,7 +782,6 @@ dependencies = [ "num-bigint", "num-integer", "num-traits", - "serde", ] [[package]] @@ -901,31 +895,6 @@ dependencies = [ "piper", ] -[[package]] -name = "bon" -version = "3.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f47dbe92550676ee653353c310dfb9cf6ba17ee70396e1f7cf0a2020ad49b2fe" -dependencies = [ - "bon-macros", - "rustversion", -] - -[[package]] -name = "bon-macros" -version = "3.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c" -dependencies = [ - "darling", - "ident_case", - "prettyplease", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.117", -] - [[package]] name = "borsh" version = "1.6.1" @@ -1681,6 +1650,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crc32fast" version = "1.5.0" @@ -1997,45 +1981,92 @@ dependencies = [ "arrow-schema", "async-trait", "bytes", + "chrono", + "datafusion-catalog 53.1.0", + "datafusion-catalog-listing 53.1.0", + "datafusion-common 53.1.0", + "datafusion-common-runtime 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-datasource-arrow 53.1.0", + "datafusion-datasource-csv 53.1.0", + "datafusion-datasource-json 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-functions 53.1.0", + "datafusion-functions-aggregate 53.1.0", + "datafusion-functions-nested 53.1.0", + "datafusion-functions-table 53.1.0", + "datafusion-functions-window 53.1.0", + "datafusion-optimizer 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-adapter 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-optimizer 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-session 53.1.0", + "datafusion-sql 53.1.0", + "futures", + "itertools 0.14.0", + "log", + "object_store 0.13.2", + "parking_lot", + "rand 0.9.4", + "regex", + "sqlparser 0.61.0", + "tempfile", + "tokio", + "url", + "uuid", +] + +[[package]] +name = "datafusion" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "997a31e15872606a49478e670c58302094c97cb96abb0a7d60720f8e92170040" +dependencies = [ + "arrow", + "arrow-schema", + "async-trait", "bzip2", "chrono", - "datafusion-catalog", - "datafusion-catalog-listing", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-datasource-arrow", + "datafusion-catalog 54.0.0", + "datafusion-catalog-listing 54.0.0", + "datafusion-common 54.0.0", + "datafusion-common-runtime 54.0.0", + "datafusion-datasource 54.0.0", + "datafusion-datasource-arrow 54.0.0", "datafusion-datasource-avro", - "datafusion-datasource-csv", - "datafusion-datasource-json", + "datafusion-datasource-csv 54.0.0", + "datafusion-datasource-json 54.0.0", "datafusion-datasource-parquet", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-nested", - "datafusion-functions-table", - "datafusion-functions-window", - "datafusion-optimizer", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-optimizer", - "datafusion-physical-plan", - "datafusion-session", - "datafusion-sql", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-expr-common 54.0.0", + "datafusion-functions 54.0.0", + "datafusion-functions-aggregate 54.0.0", + "datafusion-functions-nested 54.0.0", + "datafusion-functions-table 54.0.0", + "datafusion-functions-window 54.0.0", + "datafusion-optimizer 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-expr-adapter 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "datafusion-physical-optimizer 54.0.0", + "datafusion-physical-plan 54.0.0", + "datafusion-session 54.0.0", + "datafusion-sql 54.0.0", "flate2", "futures", + "indexmap 2.14.0", "itertools 0.14.0", "liblzma", "log", "object_store 0.13.2", "parking_lot", "parquet", - "rand 0.9.4", - "regex", - "sqlparser", + "sqlparser 0.62.0", "tempfile", "tokio", "url", @@ -2050,9 +2081,9 @@ dependencies = [ "anyhow", "clap", "custom-labels", - "datafusion", - "datafusion-common", - "datafusion-physical-plan", + "datafusion 54.0.0", + "datafusion-common 54.0.0", + "datafusion-physical-plan 54.0.0", "futures", "itertools 0.14.0", "object_store 0.13.2", @@ -2078,14 +2109,39 @@ dependencies = [ "arrow", "async-trait", "dashmap", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 53.1.0", + "datafusion-common-runtime 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-session 53.1.0", + "futures", + "itertools 0.14.0", + "log", + "object_store 0.13.2", + "parking_lot", + "tokio", +] + +[[package]] +name = "datafusion-catalog" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139" +dependencies = [ + "arrow", + "async-trait", + "dashmap", + "datafusion-common 54.0.0", + "datafusion-common-runtime 54.0.0", + "datafusion-datasource 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-plan 54.0.0", + "datafusion-session 54.0.0", "futures", "itertools 0.14.0", "log", @@ -2102,15 +2158,38 @@ checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb" dependencies = [ "arrow", "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", + "datafusion-catalog 53.1.0", + "datafusion-common 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-adapter 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", + "futures", + "itertools 0.14.0", + "log", + "object_store 0.13.2", +] + +[[package]] +name = "datafusion-catalog-listing" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897c70f871277f9ce99aa38347be0d679bbe3e617156c4d2a8378cec8a2a0891" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog 54.0.0", + "datafusion-common 54.0.0", + "datafusion-datasource 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-expr-adapter 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "datafusion-physical-plan 54.0.0", "futures", "itertools 0.14.0", "log", @@ -2124,7 +2203,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2" dependencies = [ "ahash 0.8.12", - "apache-avro", "arrow", "arrow-ipc", "chrono", @@ -2135,11 +2213,35 @@ dependencies = [ "libc", "log", "object_store 0.13.2", - "parquet", "paste", + "sqlparser 0.61.0", + "tokio", + "web-time", +] + +[[package]] +name = "datafusion-common" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c9ded5d87d9172319e006f2afdb9928d72dbacd6a90a458d8acb1e3b43a65" +dependencies = [ + "arrow", + "arrow-ipc", + "arrow-schema", + "chrono", + "foldhash 0.2.0", + "half", + "hashbrown 0.17.1", + "indexmap 2.14.0", + "itertools 0.14.0", + "libc", + "log", + "object_store 0.13.2", + "parquet", "recursive", - "sqlparser", + "sqlparser 0.62.0", "tokio", + "uuid", "web-time", ] @@ -2154,11 +2256,51 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-common-runtime" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "981b9dae74f78ee3d9f714fb49b01919eab975461b56149510c3ba9ea11287d1" +dependencies = [ + "futures", + "log", + "tokio", +] + [[package]] name = "datafusion-datasource" version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "chrono", + "datafusion-common 53.1.0", + "datafusion-common-runtime 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-adapter 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-session 53.1.0", + "futures", + "glob", + "itertools 0.14.0", + "log", + "object_store 0.13.2", + "rand 0.9.4", + "tokio", + "url", +] + +[[package]] +name = "datafusion-datasource" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffd7d295b2ec7c00d8a56562f41ed41062cf0af75549ed891c12a0a09eddfefe" dependencies = [ "arrow", "async-compression", @@ -2166,15 +2308,15 @@ dependencies = [ "bytes", "bzip2", "chrono", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 54.0.0", + "datafusion-common-runtime 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-expr-adapter 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "datafusion-physical-plan 54.0.0", + "datafusion-session 54.0.0", "flate2", "futures", "glob", @@ -2182,6 +2324,7 @@ dependencies = [ "liblzma", "log", "object_store 0.13.2", + "parking_lot", "rand 0.9.4", "tokio", "tokio-util", @@ -2199,14 +2342,38 @@ dependencies = [ "arrow-ipc", "async-trait", "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 53.1.0", + "datafusion-common-runtime 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-session 53.1.0", + "futures", + "itertools 0.14.0", + "object_store 0.13.2", + "tokio", +] + +[[package]] +name = "datafusion-datasource-arrow" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552b0b3f342f7ec41b3fbd70f6339dc82a30cfd0349e7f280e7852528085349f" +dependencies = [ + "arrow", + "arrow-ipc", + "async-trait", + "bytes", + "datafusion-common 54.0.0", + "datafusion-common-runtime 54.0.0", + "datafusion-datasource 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "datafusion-physical-plan 54.0.0", + "datafusion-session 54.0.0", "futures", "itertools 0.14.0", "object_store 0.13.2", @@ -2215,21 +2382,20 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a579c3bd290c66ea4b269493e75e8a3ed42c9c895a651f10210a29538aee50c4" +checksum = "fb517d08967d536284ce70afb5fe8583133779249f2d7b90587d339741a7f195" dependencies = [ - "apache-avro", "arrow", + "arrow-avro", "async-trait", "bytes", - "datafusion-common", - "datafusion-datasource", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 54.0.0", + "datafusion-datasource 54.0.0", + "datafusion-physical-expr-adapter 54.0.0", + "datafusion-physical-plan 54.0.0", + "datafusion-session 54.0.0", "futures", - "num-traits", "object_store 0.13.2", ] @@ -2242,14 +2408,37 @@ dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 53.1.0", + "datafusion-common-runtime 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-session 53.1.0", + "futures", + "object_store 0.13.2", + "regex", + "tokio", +] + +[[package]] +name = "datafusion-datasource-csv" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68850aa426b897e879c8b87e512ea8124f1d0a2869a4e51808ddaaddf1bc0ada" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common 54.0.0", + "datafusion-common-runtime 54.0.0", + "datafusion-datasource 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "datafusion-physical-plan 54.0.0", + "datafusion-session 54.0.0", "futures", "object_store 0.13.2", "regex", @@ -2265,14 +2454,14 @@ dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 53.1.0", + "datafusion-common-runtime 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-session 53.1.0", "futures", "object_store 0.13.2", "serde_json", @@ -2280,27 +2469,51 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "datafusion-datasource-json" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "402f93242ae08ef99139ee2c528a49d087efe88d5c7b2c3ff5480855a40ce54f" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common 54.0.0", + "datafusion-common-runtime 54.0.0", + "datafusion-datasource 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "datafusion-physical-plan 54.0.0", + "datafusion-session 54.0.0", + "futures", + "object_store 0.13.2", + "tokio", + "tokio-stream", +] + [[package]] name = "datafusion-datasource-parquet" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a8e0365e0e08e8ff94d912f0ababcf9065a1a304018ba90b1fc83c855b4997" +checksum = "ffd2499c1bee0eeccf6a57156105700eeeb17bc701899ac719183c4e74231450" dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-pruning", - "datafusion-session", + "datafusion-common 54.0.0", + "datafusion-common-runtime 54.0.0", + "datafusion-datasource 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-functions 54.0.0", + "datafusion-functions-aggregate-common 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-expr-adapter 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "datafusion-physical-plan 54.0.0", + "datafusion-pruning 54.0.0", + "datafusion-session 54.0.0", "futures", "itertools 0.14.0", "log", @@ -2316,6 +2529,12 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee" +[[package]] +name = "datafusion-doc" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb9e7e5d11130c48c8bd4e80c79a9772dd28ce6dc330baca9246205d245b9e2e" + [[package]] name = "datafusion-execution" version = "53.1.0" @@ -2327,9 +2546,31 @@ dependencies = [ "async-trait", "chrono", "dashmap", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "futures", + "log", + "object_store 0.13.2", + "parking_lot", + "rand 0.9.4", + "tempfile", + "url", +] + +[[package]] +name = "datafusion-execution" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0" +dependencies = [ + "arrow", + "arrow-buffer", + "async-trait", + "dashmap", + "datafusion-common 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-physical-expr-common 54.0.0", "futures", "log", "object_store 0.13.2", @@ -2348,18 +2589,40 @@ dependencies = [ "arrow", "async-trait", "chrono", - "datafusion-common", - "datafusion-doc", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0", + "datafusion-doc 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-functions-aggregate-common 53.1.0", + "datafusion-functions-window-common 53.1.0", + "datafusion-physical-expr-common 53.1.0", "indexmap 2.14.0", "itertools 0.14.0", "paste", + "serde_json", + "sqlparser 0.61.0", +] + +[[package]] +name = "datafusion-expr" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0" +dependencies = [ + "arrow", + "arrow-schema", + "async-trait", + "chrono", + "datafusion-common 54.0.0", + "datafusion-doc 54.0.0", + "datafusion-expr-common 54.0.0", + "datafusion-functions-aggregate-common 54.0.0", + "datafusion-functions-window-common 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "indexmap 2.14.0", + "itertools 0.14.0", "recursive", "serde_json", - "sqlparser", + "sqlparser 0.62.0", ] [[package]] @@ -2369,12 +2632,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e" dependencies = [ "arrow", - "datafusion-common", + "datafusion-common 53.1.0", "indexmap 2.14.0", "itertools 0.14.0", "paste", ] +[[package]] +name = "datafusion-expr-common" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0225491839a31b1f7d2cb8092c2d50792e2fe1c1724e4e6d08e011f5feaf4ed2" +dependencies = [ + "arrow", + "datafusion-common 54.0.0", + "indexmap 2.14.0", + "itertools 0.14.0", +] + [[package]] name = "datafusion-functions" version = "53.1.0" @@ -2388,16 +2663,16 @@ dependencies = [ "blake3", "chrono", "chrono-tz", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-macros", + "datafusion-common 53.1.0", + "datafusion-doc 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-macros 53.1.0", "hex", "itertools 0.14.0", "log", - "md-5", + "md-5 0.10.6", "memchr", "num-traits", "rand 0.9.4", @@ -2407,6 +2682,38 @@ dependencies = [ "uuid", ] +[[package]] +name = "datafusion-functions" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14872c47bfc3d21e53ec82f57074e6987a15941c1e2f43cde4ac6ae2746634e3" +dependencies = [ + "arrow", + "arrow-buffer", + "base64", + "blake2", + "blake3", + "chrono", + "chrono-tz", + "datafusion-common 54.0.0", + "datafusion-doc 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-expr-common 54.0.0", + "datafusion-macros 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "hex", + "itertools 0.14.0", + "log", + "md-5 0.11.0", + "memchr", + "num-traits", + "rand 0.9.4", + "regex", + "sha2 0.11.0", + "uuid", +] + [[package]] name = "datafusion-functions-aggregate" version = "53.1.0" @@ -2415,20 +2722,41 @@ checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad" dependencies = [ "ahash 0.8.12", "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0", + "datafusion-doc 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-functions-aggregate-common 53.1.0", + "datafusion-macros 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", "half", "log", "num-traits", "paste", ] +[[package]] +name = "datafusion-functions-aggregate" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75a2ca14e1b609be21e657e2d3130b2f446456b08393b377bb721a33952d2e09" +dependencies = [ + "arrow", + "datafusion-common 54.0.0", + "datafusion-doc 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-functions-aggregate-common 54.0.0", + "datafusion-macros 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "foldhash 0.2.0", + "half", + "log", + "num-traits", +] + [[package]] name = "datafusion-functions-aggregate-common" version = "53.1.0" @@ -2437,9 +2765,21 @@ checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47" dependencies = [ "ahash 0.8.12", "arrow", - "datafusion-common", - "datafusion-expr-common", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-physical-expr-common 53.1.0", +] + +[[package]] +name = "datafusion-functions-aggregate-common" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ece74ba09092d2ef9c9b54a38445450aea292a1f8b04faf531936b723a24b3c" +dependencies = [ + "arrow", + "datafusion-common 54.0.0", + "datafusion-expr-common 54.0.0", + "datafusion-physical-expr-common 54.0.0", ] [[package]] @@ -2450,16 +2790,16 @@ checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a" dependencies = [ "arrow", "arrow-ord", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0", + "datafusion-doc 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-functions 53.1.0", + "datafusion-functions-aggregate 53.1.0", + "datafusion-functions-aggregate-common 53.1.0", + "datafusion-macros 53.1.0", + "datafusion-physical-expr-common 53.1.0", "hashbrown 0.16.1", "itertools 0.14.0", "itoa", @@ -2467,6 +2807,31 @@ dependencies = [ "paste", ] +[[package]] +name = "datafusion-functions-nested" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3e3f9ee8ca59bf70518802107de6f1b88a9509efdc629fadc5de9d6b2d5ef5" +dependencies = [ + "arrow", + "arrow-ord", + "datafusion-common 54.0.0", + "datafusion-doc 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-expr-common 54.0.0", + "datafusion-functions 54.0.0", + "datafusion-functions-aggregate 54.0.0", + "datafusion-functions-aggregate-common 54.0.0", + "datafusion-macros 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "hashbrown 0.17.1", + "itertools 0.14.0", + "itoa", + "log", + "memchr", +] + [[package]] name = "datafusion-functions-table" version = "53.1.0" @@ -2475,14 +2840,30 @@ checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e" dependencies = [ "arrow", "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-plan", + "datafusion-catalog 53.1.0", + "datafusion-common 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-plan 53.1.0", "parking_lot", "paste", ] +[[package]] +name = "datafusion-functions-table" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89161dffc22cf2b50f9f4b1bee83b5221d3b4ed7c2e37fd7aa2b22a5297b3a26" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog 54.0.0", + "datafusion-common 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-plan 54.0.0", + "parking_lot", +] + [[package]] name = "datafusion-functions-window" version = "53.1.0" @@ -2490,25 +2871,52 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6" dependencies = [ "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-expr", - "datafusion-functions-window-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0", + "datafusion-doc 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-functions-window-common 53.1.0", + "datafusion-macros 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", "log", "paste", ] +[[package]] +name = "datafusion-functions-window" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7339345b226b3874037708bf5023ba1c2de705128f8457a095aae5ae9cb9c78" +dependencies = [ + "arrow", + "datafusion-common 54.0.0", + "datafusion-doc 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-functions-window-common 54.0.0", + "datafusion-macros 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "log", +] + [[package]] name = "datafusion-functions-window-common" version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c" dependencies = [ - "datafusion-common", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0", + "datafusion-physical-expr-common 53.1.0", +] + +[[package]] +name = "datafusion-functions-window-common" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa84836dc2392df6f43d6a29d37fb56a8ebdc8b3f4e10ae8dc15861fd20278fb" +dependencies = [ + "datafusion-common 54.0.0", + "datafusion-physical-expr-common 54.0.0", ] [[package]] @@ -2517,7 +2925,18 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd" dependencies = [ - "datafusion-doc", + "datafusion-doc 53.1.0", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "datafusion-macros" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "587164e03ad68732aa9e7bfe5686e3f25970d4c64fd4bd80790749840892dae5" +dependencies = [ + "datafusion-doc 54.0.0", "quote", "syn 2.0.117", ] @@ -2530,10 +2949,29 @@ checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace" dependencies = [ "arrow", "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", + "datafusion-common 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-physical-expr 53.1.0", + "indexmap 2.14.0", + "itertools 0.14.0", + "log", + "regex", + "regex-syntax", +] + +[[package]] +name = "datafusion-optimizer" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281" +dependencies = [ + "arrow", + "chrono", + "datafusion-common 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-expr-common 54.0.0", + "datafusion-physical-expr 54.0.0", "indexmap 2.14.0", "itertools 0.14.0", "log", @@ -2550,11 +2988,11 @@ checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59" dependencies = [ "ahash 0.8.12", "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-functions-aggregate-common 53.1.0", + "datafusion-physical-expr-common 53.1.0", "half", "hashbrown 0.16.1", "indexmap 2.14.0", @@ -2562,6 +3000,27 @@ dependencies = [ "parking_lot", "paste", "petgraph", + "tokio", +] + +[[package]] +name = "datafusion-physical-expr" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f015a4a82f6f7ff7e1d8d4bf3870a936752fa38b17705dfcc14adef95aa8922c" +dependencies = [ + "arrow", + "datafusion-common 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-expr-common 54.0.0", + "datafusion-functions-aggregate-common 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "half", + "hashbrown 0.17.1", + "indexmap 2.14.0", + "itertools 0.14.0", + "parking_lot", + "petgraph", "recursive", "tokio", ] @@ -2573,11 +3032,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4" dependencies = [ "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-functions", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-functions 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "itertools 0.14.0", +] + +[[package]] +name = "datafusion-physical-expr-adapter" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51e6ffff8acdfe54e0ea15ccf38115c4a9184433b0439f42907637928d00a235" +dependencies = [ + "arrow", + "datafusion-common 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-functions 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-expr-common 54.0.0", "itertools 0.14.0", ] @@ -2590,14 +3064,31 @@ dependencies = [ "ahash 0.8.12", "arrow", "chrono", - "datafusion-common", - "datafusion-expr-common", + "datafusion-common 53.1.0", + "datafusion-expr-common 53.1.0", "hashbrown 0.16.1", "indexmap 2.14.0", "itertools 0.14.0", "parking_lot", ] +[[package]] +name = "datafusion-physical-expr-common" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7967a3e171c6a4bf09474b3f7a14f1a3db13ed1714ba12156f33fcce2bba54e8" +dependencies = [ + "arrow", + "chrono", + "datafusion-common 54.0.0", + "datafusion-expr-common 54.0.0", + "hashbrown 0.17.1", + "indexmap 2.14.0", + "itertools 0.14.0", + "parking_lot", + "pin-project", +] + [[package]] name = "datafusion-physical-optimizer" version = "53.1.0" @@ -2605,14 +3096,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95" dependencies = [ "arrow", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-pruning", + "datafusion-common 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-pruning 53.1.0", + "itertools 0.14.0", +] + +[[package]] +name = "datafusion-physical-optimizer" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ff803e2a96054cb6d83f35f9e60fd4f42eac515e1932bd1b2dbc91d5fcbf36" +dependencies = [ + "arrow", + "datafusion-common 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-expr-common 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "datafusion-physical-plan 54.0.0", + "datafusion-pruning 54.0.0", "itertools 0.14.0", "recursive", ] @@ -2628,15 +3137,15 @@ dependencies = [ "arrow-ord", "arrow-schema", "async-trait", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0", + "datafusion-common-runtime 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-functions 53.1.0", + "datafusion-functions-aggregate-common 53.1.0", + "datafusion-functions-window-common 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", "futures", "half", "hashbrown 0.16.1", @@ -2649,6 +3158,39 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-physical-plan" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc" +dependencies = [ + "arrow", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-schema", + "async-trait", + "datafusion-common 54.0.0", + "datafusion-common-runtime 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-functions 54.0.0", + "datafusion-functions-aggregate-common 54.0.0", + "datafusion-functions-window-common 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "futures", + "half", + "hashbrown 0.17.1", + "indexmap 2.14.0", + "itertools 0.14.0", + "log", + "num-traits", + "parking_lot", + "pin-project-lite", + "tokio", +] + [[package]] name = "datafusion-pruning" version = "53.1.0" @@ -2656,16 +3198,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a" dependencies = [ "arrow", - "datafusion-common", - "datafusion-datasource", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", + "datafusion-common 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", "itertools 0.14.0", "log", ] +[[package]] +name = "datafusion-pruning" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fb9e5774660aa69c3ba93c610f175f75b65cb8c3776edb3626de8f3a4f4ee3" +dependencies = [ + "arrow", + "datafusion-common 54.0.0", + "datafusion-datasource 54.0.0", + "datafusion-expr-common 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "datafusion-physical-plan 54.0.0", + "log", +] + [[package]] name = "datafusion-session" version = "53.1.0" @@ -2673,37 +3231,54 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e" dependencies = [ "async-trait", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-plan", + "datafusion-common 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-plan 53.1.0", + "parking_lot", +] + +[[package]] +name = "datafusion-session" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15ce715fa2a61f4623cc234bcc14a3ef6a91f189128d5b14b468a6a17cdfc417" +dependencies = [ + "async-trait", + "datafusion-common 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-physical-plan 54.0.0", "parking_lot", ] [[package]] name = "datafusion-spark" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e059dcf8544da0d6598d0235be3cc29c209094a5976b2e4822e4a2cf91c2b5c5" +checksum = "390bb0bf37cb2b95ffd65eacd66f60df50793d1f94097799e416f39477a51957" dependencies = [ "arrow", "bigdecimal", "chrono", "crc32fast", - "datafusion", - "datafusion-catalog", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-nested", + "datafusion 54.0.0", + "datafusion-catalog 54.0.0", + "datafusion-common 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-functions 54.0.0", + "datafusion-functions-aggregate 54.0.0", + "datafusion-functions-aggregate-common 54.0.0", + "datafusion-functions-nested 54.0.0", "log", + "num-traits", "percent-encoding", "rand 0.9.4", "serde_json", - "sha1 0.10.6", - "sha2 0.10.9", + "sha1", + "sha2 0.11.0", + "twox-hash", "url", ] @@ -2716,29 +3291,46 @@ dependencies = [ "arrow", "bigdecimal", "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-functions-nested", + "datafusion-common 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-functions-nested 53.1.0", + "indexmap 2.14.0", + "log", + "regex", + "sqlparser 0.61.0", +] + +[[package]] +name = "datafusion-sql" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6094ad36a3ed6d7ac87b20b479b2d0b118250f66cf997603829fdc65b44a7099" +dependencies = [ + "arrow", + "bigdecimal", + "chrono", + "datafusion-common 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-functions-nested 54.0.0", "indexmap 2.14.0", "log", "recursive", "regex", - "sqlparser", + "sqlparser 0.62.0", ] [[package]] name = "datafusion-sqllogictest" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04e5a4a7a49143a68936992b6dbb0db44121c635e9992b2482817278f1e69c56" +checksum = "e0c08025966108056d3547d879c4d39e246277494f59ca12920f78187d95eea1" dependencies = [ "arrow", "async-trait", "bigdecimal", "clap", - "datafusion", + "datafusion 54.0.0", "datafusion-spark", - "datafusion-substrait", "futures", "half", "indicatif", @@ -2746,32 +3338,12 @@ dependencies = [ "log", "object_store 0.13.2", "sqllogictest", - "sqlparser", + "sqlparser 0.62.0", "tempfile", "thiserror 2.0.18", "tokio", ] -[[package]] -name = "datafusion-substrait" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98494539a5468979cc42d86c7bc5f0f8cb71ee5c742694c26fc34efdd29dd2e5" -dependencies = [ - "async-recursion", - "async-trait", - "chrono", - "datafusion", - "half", - "itertools 0.14.0", - "object_store 0.13.2", - "pbjson-types", - "prost 0.14.4", - "substrait", - "tokio", - "url", -] - [[package]] name = "deepsize" version = "0.2.0" @@ -4344,11 +4916,11 @@ dependencies = [ "chrono", "crossbeam-skiplist", "dashmap", - "datafusion", - "datafusion-expr", - "datafusion-functions", - "datafusion-physical-expr", - "datafusion-physical-plan", + "datafusion 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-functions 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-plan 53.1.0", "deepsize", "either", "futures", @@ -4453,8 +5025,8 @@ dependencies = [ "byteorder", "bytes", "chrono", - "datafusion-common", - "datafusion-sql", + "datafusion-common 53.1.0", + "datafusion-sql 53.1.0", "deepsize", "futures", "itertools 0.13.0", @@ -4494,10 +5066,10 @@ dependencies = [ "arrow-select", "async-trait", "chrono", - "datafusion", - "datafusion-common", - "datafusion-functions", - "datafusion-physical-expr", + "datafusion 53.1.0", + "datafusion-common 53.1.0", + "datafusion-functions 53.1.0", + "datafusion-physical-expr 53.1.0", "futures", "jsonb", "lance-arrow", @@ -4587,7 +5159,7 @@ dependencies = [ "async-trait", "byteorder", "bytes", - "datafusion-common", + "datafusion-common 53.1.0", "deepsize", "futures", "lance-arrow", @@ -4625,11 +5197,11 @@ dependencies = [ "bytes", "chrono", "crossbeam-queue", - "datafusion", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-sql", + "datafusion 53.1.0", + "datafusion-common 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-sql 53.1.0", "deepsize", "dirs", "fst", @@ -5229,6 +5801,16 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + [[package]] name = "memchr" version = "2.8.1" @@ -5536,7 +6118,6 @@ checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", "num-traits", - "serde", ] [[package]] @@ -5707,7 +6288,7 @@ dependencies = [ "humantime", "hyper", "itertools 0.14.0", - "md-5", + "md-5 0.10.6", "parking_lot", "percent-encoding", "quick-xml", @@ -6063,43 +6644,6 @@ dependencies = [ "stfu8", ] -[[package]] -name = "pbjson" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "898bac3fa00d0ba57a4e8289837e965baa2dee8c3749f3b11d45a64b4223d9c3" -dependencies = [ - "base64", - "serde", -] - -[[package]] -name = "pbjson-build" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af22d08a625a2213a78dbb0ffa253318c5c79ce3133d32d296655a7bdfb02095" -dependencies = [ - "heck", - "itertools 0.14.0", - "prost 0.14.4", - "prost-types", -] - -[[package]] -name = "pbjson-types" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e748e28374f10a330ee3bb9f29b828c0ac79831a32bab65015ad9b661ead526" -dependencies = [ - "bytes", - "chrono", - "pbjson", - "pbjson-build", - "prost 0.14.4", - "prost-build", - "serde", -] - [[package]] name = "pbkdf2" version = "0.13.0" @@ -6606,12 +7150,6 @@ dependencies = [ "url", ] -[[package]] -name = "quad-rand" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" - [[package]] name = "quick-xml" version = "0.39.4" @@ -7058,16 +7596,6 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" -[[package]] -name = "regress" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2057b2325e68a893284d1538021ab90279adac1139957ca2a74426c6f118fb48" -dependencies = [ - "hashbrown 0.16.1", - "memchr", -] - [[package]] name = "relative-path" version = "1.9.3" @@ -7447,18 +7975,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "schemars" -version = "0.8.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615" -dependencies = [ - "dyn-clone", - "schemars_derive", - "serde", - "serde_json", -] - [[package]] name = "schemars" version = "0.9.0" @@ -7483,18 +7999,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "schemars_derive" -version = "0.8.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32e265784ad618884abaea0600a9adf15393368d840e0222d101a072f3f7534d" -dependencies = [ - "proc-macro2", - "quote", - "serde_derive_internals", - "syn 2.0.117", -] - [[package]] name = "scoped-tls" version = "1.0.1" @@ -7568,16 +8072,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde_bytes" -version = "0.11.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" -dependencies = [ - "serde", - "serde_core", -] - [[package]] name = "serde_core" version = "1.0.228" @@ -7598,17 +8092,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "serde_derive_internals" -version = "0.29.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "serde_json" version = "1.0.150" @@ -7662,18 +8145,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_tokenstream" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7c49585c52c01f13c5c2ebb333f14f6885d76daa768d8a037d28017ec538c69" -dependencies = [ - "proc-macro2", - "quote", - "serde", - "syn 2.0.117", -] - [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -7718,19 +8189,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "serde_yaml" -version = "0.9.34+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" -dependencies = [ - "indexmap 2.14.0", - "itoa", - "ryu", - "serde", - "unsafe-libyaml", -] - [[package]] name = "serial_test" version = "3.5.0" @@ -7756,17 +8214,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "sha1" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest 0.10.7", -] - [[package]] name = "sha1" version = "0.11.0" @@ -8004,7 +8451,7 @@ dependencies = [ "humantime", "itertools 0.13.0", "libtest-mimic", - "md-5", + "md-5 0.10.6", "owo-colors", "rand 0.8.6", "regex", @@ -8020,6 +8467,16 @@ name = "sqlparser" version = "0.61.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7" +dependencies = [ + "log", + "sqlparser_derive", +] + +[[package]] +name = "sqlparser" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" dependencies = [ "log", "recursive", @@ -8164,31 +8621,6 @@ dependencies = [ "unicode-width 0.1.14", ] -[[package]] -name = "substrait" -version = "0.62.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62fc4b483a129b9772ccb9c3f7945a472112fdd9140da87f8a4e7f1d44e045d0" -dependencies = [ - "heck", - "pbjson", - "pbjson-build", - "pbjson-types", - "prettyplease", - "prost 0.14.4", - "prost-build", - "prost-types", - "regress", - "schemars 0.8.22", - "semver", - "serde", - "serde_json", - "serde_yaml", - "syn 2.0.117", - "typify", - "walkdir", -] - [[package]] name = "subtle" version = "2.6.1" @@ -8893,53 +9325,6 @@ version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" -[[package]] -name = "typify" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d5bcc6f62eb1fa8aa4098f39b29f93dcb914e17158b76c50360911257aa629" -dependencies = [ - "typify-impl", - "typify-macro", -] - -[[package]] -name = "typify-impl" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1eb359f7ffa4f9ebe947fa11a1b2da054564502968db5f317b7e37693cb2240" -dependencies = [ - "heck", - "log", - "proc-macro2", - "quote", - "regress", - "schemars 0.8.22", - "semver", - "serde", - "serde_json", - "syn 2.0.117", - "thiserror 2.0.18", - "unicode-ident", -] - -[[package]] -name = "typify-macro" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "911c32f3c8514b048c1b228361bebb5e6d73aeec01696e8cc0e82e2ffef8ab7a" -dependencies = [ - "proc-macro2", - "quote", - "schemars 0.8.22", - "semver", - "serde", - "serde_json", - "serde_tokenstream", - "syn 2.0.117", - "typify-impl", -] - [[package]] name = "uint" version = "0.10.0" @@ -9015,12 +9400,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" -[[package]] -name = "unsafe-libyaml" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" - [[package]] name = "untrusted" version = "0.9.0" @@ -9559,19 +9938,19 @@ dependencies = [ "anyhow", "arrow-schema", "async-trait", - "datafusion", - "datafusion-catalog", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-pruning", + "datafusion 54.0.0", + "datafusion-catalog 54.0.0", + "datafusion-common 54.0.0", + "datafusion-common-runtime 54.0.0", + "datafusion-datasource 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-functions 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-expr-adapter 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "datafusion-physical-plan 54.0.0", + "datafusion-pruning 54.0.0", "futures", "insta", "itertools 0.14.0", @@ -10151,7 +10530,7 @@ dependencies = [ "async-trait", "bigdecimal", "clap", - "datafusion", + "datafusion 54.0.0", "datafusion-sqllogictest", "futures", "indicatif", @@ -10210,7 +10589,7 @@ dependencies = [ "clap", "console_error_panic_hook", "crossterm 0.29.0", - "datafusion", + "datafusion 54.0.0", "env_logger", "flatbuffers", "futures", @@ -11152,7 +11531,7 @@ dependencies = [ "memchr", "pbkdf2", "ppmd-rust", - "sha1 0.11.0", + "sha1", "time", "typed-path", "zeroize", diff --git a/Cargo.toml b/Cargo.toml index 7c87eb158f3..35cc2549a84 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -99,17 +99,17 @@ anyhow = "1.0.97" arbitrary = "1.3.2" arc-swap = "1.9" arcref = "0.2.0" -arrow-arith = "58" -arrow-array = "58" -arrow-buffer = "58" -arrow-cast = "58" -arrow-data = "58" -arrow-ipc = "58" -arrow-ord = "58" -arrow-row = "58" -arrow-schema = "58" -arrow-select = "58" -arrow-string = "58" +arrow-arith = "58.3" +arrow-array = "58.3" +arrow-buffer = "58.3" +arrow-cast = "58.3" +arrow-data = "58.3" +arrow-ipc = "58.3" +arrow-ord = "58.3" +arrow-row = "58.3" +arrow-schema = "58.3" +arrow-select = "58.3" +arrow-string = "58.3" async-fs = "2.2.0" async-lock = "3.4" async-stream = "0.3.6" @@ -137,20 +137,20 @@ cudarc = { version = "0.19.0", features = [ ] } custom-labels = "0.4.4" dashmap = "6.1.0" -datafusion = { version = "53", default-features = false, features = ["sql"] } -datafusion-catalog = { version = "53" } -datafusion-common = { version = "53" } -datafusion-common-runtime = { version = "53" } -datafusion-datasource = { version = "53", default-features = false } -datafusion-execution = { version = "53" } -datafusion-expr = { version = "53" } -datafusion-functions = { version = "53" } -datafusion-physical-expr = { version = "53" } -datafusion-physical-expr-adapter = { version = "53" } -datafusion-physical-expr-common = { version = "53" } -datafusion-physical-plan = { version = "53" } -datafusion-pruning = { version = "53" } -datafusion-sqllogictest = { version = "53" } +datafusion = { version = "54", default-features = false, features = ["sql"] } +datafusion-catalog = { version = "54" } +datafusion-common = { version = "54" } +datafusion-common-runtime = { version = "54" } +datafusion-datasource = { version = "54", default-features = false } +datafusion-execution = { version = "54" } +datafusion-expr = { version = "54" } +datafusion-functions = { version = "54" } +datafusion-physical-expr = { version = "54" } +datafusion-physical-expr-adapter = { version = "54" } +datafusion-physical-expr-common = { version = "54" } +datafusion-physical-plan = { version = "54" } +datafusion-pruning = { version = "54" } +datafusion-sqllogictest = { version = "54" } dirs = "6.0.0" divan = { package = "codspeed-divan-compat", version = "4.0.4" } enum-iterator = "2.0.0" @@ -164,7 +164,7 @@ get_dir = "0.5.0" glob = "0.3.2" goldenfile = "1" half = { version = "2.7.1", features = ["std", "num-traits"] } -hashbrown = "0.17.0" +hashbrown = "0.17.1" humansize = "2.1.3" indicatif = "0.18.0" insta = "1.43" @@ -190,7 +190,7 @@ noodles-bgzf = "0.47.0" noodles-vcf = { version = "0.88.0", features = ["async"] } num-traits = "0.2.19" num_enum = { version = "0.7.3", default-features = false } -object_store = { version = "0.13.1", default-features = false } +object_store = { version = "0.13.2", default-features = false } once_cell = "1.21" oneshot = { version = "0.2.0", features = ["async"] } onpair = { version = "0.0.4" } @@ -198,9 +198,9 @@ opentelemetry = "0.32.0" opentelemetry-otlp = "0.32.0" opentelemetry_sdk = "0.32.0" parking_lot = { version = "0.12.3", features = ["nightly"] } -parquet = "58" -parquet-variant = "58" -parquet-variant-compute = "58" +parquet = "58.3" +parquet-variant = "58.3" +parquet-variant-compute = "58.3" paste = "1.0.15" pco = "1.0.1" pin-project-lite = "0.2.15" @@ -253,7 +253,7 @@ tempfile = "3" termtree = { version = "1.0" } test-with = "0.16" thiserror = "2.0.3" -tokio = { version = "1.48" } +tokio = { version = "1.52" } tokio-stream = "0.1.17" tokio-util = "0.7.17" vortex-array-macros = { version = "0.1.0", path = "./vortex-array-macros" } @@ -264,7 +264,7 @@ tracing = { version = "0.1.41", default-features = false } tracing-perfetto = "0.1.5" tracing-subscriber = "0.3" url = "2.5.7" -uuid = { version = "1.21", features = ["js"] } +uuid = { version = "1.23", features = ["js"] } wasm-bindgen-futures = "0.4.54" xshell = "0.2.6" zigzag = "0.1.0" diff --git a/benchmarks/datafusion-bench/src/lib.rs b/benchmarks/datafusion-bench/src/lib.rs index 469c9be6177..c8353eb1f85 100644 --- a/benchmarks/datafusion-bench/src/lib.rs +++ b/benchmarks/datafusion-bench/src/lib.rs @@ -14,7 +14,7 @@ use datafusion::datasource::provider::DefaultTableFactory; use datafusion::execution::SessionStateBuilder; use datafusion::execution::cache::DefaultListFilesCache; use datafusion::execution::cache::cache_manager::CacheManagerConfig; -use datafusion::execution::cache::cache_unit::DefaultFileStatisticsCache; +use datafusion::execution::cache::file_statistics_cache::DefaultFileStatisticsCache; use datafusion::execution::runtime_env::RuntimeEnvBuilder; use datafusion::prelude::SessionConfig; use datafusion::prelude::SessionContext; @@ -37,7 +37,7 @@ pub fn get_session_context() -> SessionContext { let file_static_cache = Arc::new(DefaultFileStatisticsCache::default()); let list_file_cache = Arc::new(DefaultListFilesCache::default()); let cache_config = CacheManagerConfig::default() - .with_files_statistics_cache(Some(file_static_cache)) + .with_file_statistics_cache(Some(file_static_cache)) .with_list_files_cache(Some(list_file_cache)); rt_builder = rt_builder.with_cache_manager(cache_config); diff --git a/benchmarks/datafusion-bench/src/main.rs b/benchmarks/datafusion-bench/src/main.rs index 52046a03da3..b8f9ac42df6 100644 --- a/benchmarks/datafusion-bench/src/main.rs +++ b/benchmarks/datafusion-bench/src/main.rs @@ -278,7 +278,14 @@ async fn register_benchmark_tables( None => config.infer_schema(&session.state()).await?, }; - let listing_table = Arc::new(ListingTable::try_new(config)?); + let listing_table = Arc::new( + ListingTable::try_new(config)?.with_cache( + session + .runtime_env() + .cache_manager + .get_file_statistic_cache(), + ), + ); session.register_table(table.name, listing_table)?; } diff --git a/vortex-datafusion/src/convert/exprs.rs b/vortex-datafusion/src/convert/exprs.rs index 6a9ac0fdc52..b6d741fc5ec 100644 --- a/vortex-datafusion/src/convert/exprs.rs +++ b/vortex-datafusion/src/convert/exprs.rs @@ -127,7 +127,6 @@ impl DefaultExpressionConvertor { let mut result = self.convert(source_expr.as_ref())?; for expr in field_names { let field_name = expr - .as_any() .downcast_ref::() .ok_or_else(|| exec_datafusion_err!("get_field field name must be a literal"))? .value() @@ -195,7 +194,7 @@ impl ExpressionConvertor for DefaultExpressionConvertor { fn convert(&self, df: &dyn PhysicalExpr) -> DFResult { // TODO(joe): Don't return an error when we have an unsupported node, bubble up "TRUE" as in keep // for that node, up to any `and` or `or` node. - if let Some(binary_expr) = df.as_any().downcast_ref::() { + if let Some(binary_expr) = df.downcast_ref::() { let left = self.convert(binary_expr.left().as_ref())?; let right = self.convert(binary_expr.right().as_ref())?; let operator = try_operator_from_df(binary_expr.op())?; @@ -203,11 +202,11 @@ impl ExpressionConvertor for DefaultExpressionConvertor { return Ok(Binary.new_expr(operator, [left, right])); } - if let Some(col_expr) = df.as_any().downcast_ref::() { + if let Some(col_expr) = df.downcast_ref::() { return Ok(get_item(col_expr.name().to_owned(), root())); } - if let Some(like) = df.as_any().downcast_ref::() { + if let Some(like) = df.downcast_ref::() { let child = self.convert(like.expr().as_ref())?; let pattern = self.convert(like.pattern().as_ref())?; return Ok(Like.new_expr( @@ -219,42 +218,34 @@ impl ExpressionConvertor for DefaultExpressionConvertor { )); } - if let Some(literal) = df.as_any().downcast_ref::() { + if let Some(literal) = df.downcast_ref::() { let value = Scalar::from_df(literal.value()); return Ok(lit(value)); } - if let Some(cast_expr) = df.as_any().downcast_ref::() { - let cast_dtype = DType::from_arrow((cast_expr.cast_type(), Nullability::Nullable)); + if let Some(cast_expr) = df.downcast_ref::() { + let cast_dtype = DType::from_arrow(cast_expr.target_field().as_ref()); let child = self.convert(cast_expr.expr().as_ref())?; return Ok(cast(child, cast_dtype)); } - if let Some(cast_col_expr) = df.as_any().downcast_ref::() { - let target = cast_col_expr.target_field(); - - let target_dtype = DType::from_arrow((target.data_type(), target.is_nullable().into())); - let child = self.convert(cast_col_expr.expr().as_ref())?; - return Ok(cast(child, target_dtype)); - } - - if let Some(is_null_expr) = df.as_any().downcast_ref::() { + if let Some(is_null_expr) = df.downcast_ref::() { let arg = self.convert(is_null_expr.arg().as_ref())?; return Ok(is_null(arg)); } - if let Some(is_not_null_expr) = df.as_any().downcast_ref::() { + if let Some(is_not_null_expr) = df.downcast_ref::() { let arg = self.convert(is_not_null_expr.arg().as_ref())?; return Ok(is_not_null(arg)); } - if let Some(in_list) = df.as_any().downcast_ref::() { + if let Some(in_list) = df.downcast_ref::() { let value = self.convert(in_list.expr().as_ref())?; let list_elements: Vec<_> = in_list .list() .iter() .map(|e| { - if let Some(lit) = e.as_any().downcast_ref::() { + if let Some(lit) = e.downcast_ref::() { Ok(Scalar::from_df(lit.value())) } else { Err(exec_datafusion_err!("Failed to cast sub-expression")) @@ -272,11 +263,11 @@ impl ExpressionConvertor for DefaultExpressionConvertor { return Ok(if in_list.negated() { not(expr) } else { expr }); } - if let Some(scalar_fn) = df.as_any().downcast_ref::() { + if let Some(scalar_fn) = df.downcast_ref::() { return self.try_convert_scalar_function(scalar_fn); } - if let Some(case_expr) = df.as_any().downcast_ref::() { + if let Some(case_expr) = df.downcast_ref::() { return self.try_convert_case_expr(case_expr); } @@ -297,7 +288,7 @@ impl ExpressionConvertor for DefaultExpressionConvertor { for projection_expr in source_projection.iter() { let r = projection_expr.expr.apply(|node| { // We only pull column children of scalar functions that we can't push into the scan. - if let Some(scalar_fn_expr) = node.as_any().downcast_ref::() + if let Some(scalar_fn_expr) = node.downcast_ref::() && !can_scalar_fn_be_pushed_down(scalar_fn_expr) { scan_projection.extend( @@ -312,7 +303,7 @@ impl ExpressionConvertor for DefaultExpressionConvertor { // DataFusion assumes different decimal types can be coerced. // Vortex expects a perfect match so we don't push it down. - if let Some(binary_expr) = node.as_any().downcast_ref::() + if let Some(binary_expr) = node.downcast_ref::() && binary_expr.op().is_numerical_operators() && (is_decimal(&binary_expr.left().data_type(input_schema)?) && is_decimal(&binary_expr.right().data_type(input_schema)?)) @@ -406,14 +397,13 @@ fn try_operator_from_df(value: &DFOperator) -> DFResult { } } -fn can_be_pushed_down_impl(df_expr: &Arc, schema: &Schema) -> bool { +fn can_be_pushed_down_impl(expr: &Arc, schema: &Schema) -> bool { // We currently do not support pushdown of dynamic expressions in DF. // See issue: https://github.com/vortex-data/vortex/issues/4034 - if is_dynamic_physical_expr(df_expr) { + if is_dynamic_physical_expr(expr) { return false; } - let expr = df_expr.as_any(); if let Some(binary) = expr.downcast_ref::() { can_binary_be_pushed_down(binary, schema) } else if let Some(col) = expr.downcast_ref::() { @@ -429,9 +419,6 @@ fn can_be_pushed_down_impl(df_expr: &Arc, schema: &Schema) -> } else if let Some(cast_expr) = expr.downcast_ref::() { // CastExpr child must be an expression type that convert() can handle is_convertible_expr(cast_expr.expr()) - } else if let Some(cast_col_expr) = expr.downcast_ref::() { - // CastColumnExpr child must be an expression type that convert() can handle - is_convertible_expr(cast_col_expr.expr()) } else if let Some(is_null) = expr.downcast_ref::() { can_be_pushed_down_impl(is_null.arg(), schema) } else if let Some(is_not_null) = expr.downcast_ref::() { @@ -447,7 +434,7 @@ fn can_be_pushed_down_impl(df_expr: &Arc, schema: &Schema) -> } else if let Some(case_expr) = expr.downcast_ref::() { can_case_be_pushed_down(case_expr, schema) } else { - tracing::debug!(%df_expr, "DataFusion expression can't be pushed down"); + tracing::debug!(%expr, "DataFusion expression can't be pushed down"); false } } @@ -455,9 +442,7 @@ fn can_be_pushed_down_impl(df_expr: &Arc, schema: &Schema) -> /// Checks if an expression type is one that convert() can handle. /// This is less restrictive than can_be_pushed_down since it only checks /// expression types, not data type support. -fn is_convertible_expr(df_expr: &Arc) -> bool { - let expr = df_expr.as_any(); - +fn is_convertible_expr(expr: &Arc) -> bool { // Expression types that convert() handles expr.downcast_ref::().is_some() || expr.downcast_ref::().is_some() @@ -466,9 +451,6 @@ fn is_convertible_expr(df_expr: &Arc) -> bool { || expr .downcast_ref::() .is_some_and(|e| is_convertible_expr(e.expr())) - || expr - .downcast_ref::() - .is_some_and(|e| is_convertible_expr(e.expr())) || expr.downcast_ref::().is_some() || expr.downcast_ref::().is_some() || expr.downcast_ref::().is_some() @@ -568,6 +550,8 @@ mod tests { use arrow_schema::Field; use arrow_schema::Schema; use arrow_schema::TimeUnit as ArrowTimeUnit; + use datafusion::arrow::array::AsArray; + use datafusion::arrow::datatypes::Int32Type; use datafusion_common::ScalarValue; use datafusion_expr::Operator as DFOperator; use datafusion_physical_expr::PhysicalExpr; @@ -988,12 +972,7 @@ mod tests { let vortex_as_arrow = vortex_result.into_primitive().as_slice::().to_vec(); // Convert DataFusion result to Vec for comparison - let df_as_arrow: Vec = df_array - .as_any() - .downcast_ref::() - .unwrap() - .values() - .to_vec(); + let df_as_arrow: Vec = df_array.as_primitive::().values().to_vec(); // Compare results // Expected: [0, 0, 50, 100, 100] for values [1, 5, 10, 15, 20] diff --git a/vortex-datafusion/src/persistent/format.rs b/vortex-datafusion/src/persistent/format.rs index 2ccfbf8ca7b..964d6ad19e1 100644 --- a/vortex-datafusion/src/persistent/format.rs +++ b/vortex-datafusion/src/persistent/format.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use std::any::Any; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; @@ -299,10 +298,6 @@ impl FileFormatFactory for VortexFormatFactory { fn default(&self) -> Arc { Arc::new(VortexFormat::new(self.session.clone())) } - - fn as_any(&self) -> &dyn Any { - self - } } impl VortexFormat { @@ -330,10 +325,6 @@ impl VortexFormat { #[async_trait] impl FileFormat for VortexFormat { - fn as_any(&self) -> &dyn Any { - self - } - fn compression_type(&self) -> Option { None } @@ -594,7 +585,6 @@ impl FileFormat for VortexFormat { ) -> DFResult> { let mut source = file_scan_config .file_source() - .as_any() .downcast_ref::() .cloned() .ok_or_else(|| internal_datafusion_err!("Expected VortexSource"))?; diff --git a/vortex-datafusion/src/persistent/metrics.rs b/vortex-datafusion/src/persistent/metrics.rs index d4c6456d6fa..e3bb1b18868 100644 --- a/vortex-datafusion/src/persistent/metrics.rs +++ b/vortex-datafusion/src/persistent/metrics.rs @@ -73,16 +73,13 @@ impl VortexMetricsFinder { impl ExecutionPlanVisitor for VortexMetricsFinder { type Error = std::convert::Infallible; fn pre_visit(&mut self, plan: &dyn ExecutionPlan) -> Result { - if let Some(exec) = plan.as_any().downcast_ref::() { + if let Some(exec) = plan.downcast_ref::() { // Start with exec metrics or create a new set let mut set = exec.metrics().unwrap_or_default(); // Include our own metrics from VortexSource - if let Some(file_scan) = exec.data_source().as_any().downcast_ref::() - && let Some(scan) = file_scan - .file_source - .as_any() - .downcast_ref::() + if let Some(file_scan) = exec.data_source().downcast_ref::() + && let Some(scan) = file_scan.file_source.downcast_ref::() { for metric in scan .metrics_registry() @@ -240,7 +237,7 @@ mod tests { &mut self, plan: &dyn datafusion_physical_plan::ExecutionPlan, ) -> Result { - if plan.as_any().downcast_ref::().is_some() { + if plan.downcast_ref::().is_some() { self.0 += 1; Ok(false) } else { diff --git a/vortex-datafusion/src/persistent/opener.rs b/vortex-datafusion/src/persistent/opener.rs index 8a579c8557d..af3df1f72b8 100644 --- a/vortex-datafusion/src/persistent/opener.rs +++ b/vortex-datafusion/src/persistent/opener.rs @@ -314,9 +314,7 @@ impl FileOpener for VortexOpener { let mut scan_builder = ScanBuilder::new(session.clone(), Arc::clone(&layout_reader)); - if let Some(extensions) = file.extensions - && let Some(vortex_plan) = extensions.downcast_ref::() - { + if let Some(vortex_plan) = file.extensions.get::() { scan_builder = vortex_plan.apply_to_builder(scan_builder); } @@ -1149,9 +1147,12 @@ mod tests { let schema = batch.schema(); let mut file = PartitionedFile::new(file_path.to_string(), data_size); - file.extensions = Some(Arc::new(VortexAccessPlan::default().with_selection( - Selection::IncludeByIndex(Buffer::from_iter(vec![1, 3, 5, 7])), - ))); + file.extensions + .insert( + VortexAccessPlan::default().with_selection(Selection::IncludeByIndex( + Buffer::from_iter(vec![1, 3, 5, 7]), + )), + ); let opener = make_test_opener( Arc::clone(&object_store), @@ -1190,9 +1191,12 @@ mod tests { let schema = batch.schema(); let mut file = PartitionedFile::new(file_path.to_string(), data_size); - file.extensions = Some(Arc::new(VortexAccessPlan::default().with_selection( - Selection::ExcludeByIndex(Buffer::from_iter(vec![0, 2, 4, 6, 8])), - ))); + file.extensions + .insert( + VortexAccessPlan::default().with_selection(Selection::ExcludeByIndex( + Buffer::from_iter(vec![0, 2, 4, 6, 8]), + )), + ); let opener = make_test_opener( Arc::clone(&object_store), @@ -1234,9 +1238,8 @@ mod tests { let schema = batch.schema(); let mut file = PartitionedFile::new(file_path.to_string(), data_size); - file.extensions = Some(Arc::new( - VortexAccessPlan::default().with_selection(Selection::All), - )); + file.extensions + .insert(VortexAccessPlan::default().with_selection(Selection::All)); let opener = make_test_opener( Arc::clone(&object_store), diff --git a/vortex-datafusion/src/persistent/sink.rs b/vortex-datafusion/src/persistent/sink.rs index b05cbe84215..51cb60953da 100644 --- a/vortex-datafusion/src/persistent/sink.rs +++ b/vortex-datafusion/src/persistent/sink.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use std::any::Any; use std::sync::Arc; use arrow_schema::SchemaRef; @@ -69,10 +68,6 @@ impl DisplayAs for VortexSink { #[async_trait] impl DataSink for VortexSink { - fn as_any(&self) -> &dyn Any { - self - } - fn metrics(&self) -> Option { None } diff --git a/vortex-datafusion/src/persistent/source.rs b/vortex-datafusion/src/persistent/source.rs index aed2024d5d9..f8ad847d5e7 100644 --- a/vortex-datafusion/src/persistent/source.rs +++ b/vortex-datafusion/src/persistent/source.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use std::any::Any; use std::fmt::Formatter; use std::ops::Range; use std::sync::Arc; @@ -361,10 +360,6 @@ impl FileSource for VortexSource { )?)) } - fn as_any(&self) -> &dyn Any { - self - } - fn with_batch_size(&self, batch_size: usize) -> Arc { let mut source = self.clone(); source.batch_size = Some(batch_size); diff --git a/vortex-datafusion/src/v2/source.rs b/vortex-datafusion/src/v2/source.rs index ffb7c068c1e..82fae9bf56d 100644 --- a/vortex-datafusion/src/v2/source.rs +++ b/vortex-datafusion/src/v2/source.rs @@ -67,7 +67,6 @@ //! [`DataSourceRef`]: vortex::scan::DataSourceRef //! [`ScanRequest`]: vortex::scan::ScanRequest -use std::any::Any; use std::fmt; use std::fmt::Formatter; use std::sync::Arc; @@ -469,10 +468,6 @@ impl DataSource for VortexDataSource { ))) } - fn as_any(&self) -> &dyn Any { - self - } - fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> fmt::Result { write!( f, @@ -509,7 +504,7 @@ impl DataSource for VortexDataSource { EquivalenceProperties::new(Arc::clone(&self.leftover_schema)) } - fn partition_statistics(&self, _partition: Option) -> DFResult { + fn partition_statistics(&self, _partition: Option) -> DFResult> { // FIXME(ngates): this should be adjusted based on filters. See DuckDB for heuristics, // and in the future, store the selectivity stats in the session. let num_rows = estimate_to_df_precision(&self.data_source.row_count()); @@ -521,11 +516,11 @@ impl DataSource for VortexDataSource { // from the initial schema after try_swapping_with_projection adds computed columns. let column_statistics = self.leftover_statistics.clone(); - Ok(Statistics { + Ok(Arc::new(Statistics { num_rows, total_byte_size, column_statistics, - }) + })) } fn with_fetch(&self, limit: Option) -> Option> { diff --git a/vortex-datafusion/src/v2/table.rs b/vortex-datafusion/src/v2/table.rs index c45a25c8b91..b881eb025ab 100644 --- a/vortex-datafusion/src/v2/table.rs +++ b/vortex-datafusion/src/v2/table.rs @@ -7,7 +7,6 @@ //! [`DataSourceRef`]: vortex::scan::DataSourceRef //! [`TableProvider`]: datafusion_catalog::TableProvider -use std::any::Any; use std::fmt; use std::sync::Arc; @@ -107,10 +106,6 @@ impl VortexTable { #[async_trait] impl TableProvider for VortexTable { - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { Arc::clone(&self.arrow_schema) }