Skip to content

Commit 1ac8ba9

Browse files
committed
Merge remote-tracking branch 'upstreamDF/main' into lia/use-arrow-pool-to-fix-memory-overaccounting-aggregations
2 parents cb09c94 + 3420a2d commit 1ac8ba9

File tree

175 files changed

+8767
-2124
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

175 files changed

+8767
-2124
lines changed

.github/workflows/audit.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ jobs:
4242
steps:
4343
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
4444
- name: Install cargo-audit
45-
uses: taiki-e/install-action@b9c5db3aef04caffaf95a1d03931de10fb2a140f # v2.65.1
45+
uses: taiki-e/install-action@ff581034fb69296c525e51afd68cb9823bfbe4ed # v2.65.8
4646
with:
4747
tool: cargo-audit
4848
- name: Run audit check

.github/workflows/rust.yml

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -421,7 +421,7 @@ jobs:
421421
sudo apt-get update -qq
422422
sudo apt-get install -y -qq clang
423423
- name: Setup wasm-pack
424-
uses: taiki-e/install-action@b9c5db3aef04caffaf95a1d03931de10fb2a140f # v2.65.1
424+
uses: taiki-e/install-action@ff581034fb69296c525e51afd68cb9823bfbe4ed # v2.65.8
425425
with:
426426
tool: wasm-pack
427427
- name: Run tests with headless mode
@@ -709,6 +709,23 @@ jobs:
709709
./dev/update_function_docs.sh
710710
git diff --exit-code
711711
712+
examples-docs-check:
713+
name: check example README is up-to-date
714+
needs: linux-build-lib
715+
runs-on: ubuntu-latest
716+
container:
717+
image: amd64/rust
718+
719+
steps:
720+
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
721+
with:
722+
submodules: true
723+
fetch-depth: 1
724+
725+
- name: Run examples docs check script
726+
run: |
727+
bash ci/scripts/check_examples_docs.sh
728+
712729
# Verify MSRV for the crates which are directly used by other projects:
713730
# - datafusion
714731
# - datafusion-substrait
@@ -724,7 +741,7 @@ jobs:
724741
- name: Setup Rust toolchain
725742
uses: ./.github/actions/setup-builder
726743
- name: Install cargo-msrv
727-
uses: taiki-e/install-action@b9c5db3aef04caffaf95a1d03931de10fb2a140f # v2.65.1
744+
uses: taiki-e/install-action@ff581034fb69296c525e51afd68cb9823bfbe4ed # v2.65.8
728745
with:
729746
tool: cargo-msrv
730747

Cargo.lock

Lines changed: 25 additions & 15 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ flate2 = "1.1.5"
156156
futures = "0.3"
157157
glob = "0.3.0"
158158
half = { version = "2.7.0", default-features = false }
159-
hashbrown = { version = "0.14.5", features = ["raw"] }
159+
hashbrown = { version = "0.16.1" }
160160
hex = { version = "0.4.3" }
161161
indexmap = "2.12.1"
162162
insta = { version = "1.45.0", features = ["glob", "filters"] }

ci/scripts/check_examples_docs.sh

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
20+
set -euo pipefail
21+
22+
EXAMPLES_DIR="datafusion-examples/examples"
23+
README="datafusion-examples/README.md"
24+
25+
# ffi examples are skipped because they were not part of the recent example
26+
# consolidation work and do not follow the new grouping and execution pattern.
27+
# They are not documented in the README using the new structure, so including
28+
# them here would cause false CI failures.
29+
SKIP_LIST=("ffi")
30+
31+
missing=0
32+
33+
skip() {
34+
local value="$1"
35+
for item in "${SKIP_LIST[@]}"; do
36+
if [[ "$item" == "$value" ]]; then
37+
return 0
38+
fi
39+
done
40+
return 1
41+
}
42+
43+
# collect folder names
44+
folders=$(find "$EXAMPLES_DIR" -mindepth 1 -maxdepth 1 -type d -exec basename {} \;)
45+
46+
# collect group names from README headers
47+
groups=$(grep "^### Group:" "$README" | sed -E 's/^### Group: `([^`]+)`.*/\1/')
48+
49+
for folder in $folders; do
50+
if skip "$folder"; then
51+
echo "Skipped group: $folder"
52+
continue
53+
fi
54+
55+
if ! echo "$groups" | grep -qx "$folder"; then
56+
echo "Missing README entry for example group: $folder"
57+
missing=1
58+
fi
59+
done
60+
61+
if [[ $missing -eq 1 ]]; then
62+
echo "README is out of sync with examples"
63+
exit 1
64+
fi

datafusion-examples/README.md

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -191,12 +191,12 @@ cargo run --example dataframe -- dataframe
191191

192192
#### Category: Single Process
193193

194-
| Subcommand | File Path | Description |
195-
| ---------- | ----------------------------------------------------- | ------------------------------ |
196-
| analysis | [`sql_ops/analysis.rs`](examples/sql_ops/analysis.rs) | Analyze SQL queries |
197-
| dialect | [`sql_ops/dialect.rs`](examples/sql_ops/dialect.rs) | Implement a custom SQL dialect |
198-
| frontend | [`sql_ops/frontend.rs`](examples/sql_ops/frontend.rs) | Build LogicalPlans from SQL |
199-
| query | [`sql_ops/query.rs`](examples/sql_ops/query.rs) | Query data using SQL |
194+
| Subcommand | File Path | Description |
195+
| ----------------- | ----------------------------------------------------------------------- | -------------------------------------------------- |
196+
| analysis | [`sql_ops/analysis.rs`](examples/sql_ops/analysis.rs) | Analyze SQL queries |
197+
| custom_sql_parser | [`sql_ops/custom_sql_parser.rs`](examples/sql_ops/custom_sql_parser.rs) | Implement a custom SQL parser to extend DataFusion |
198+
| frontend | [`sql_ops/frontend.rs`](examples/sql_ops/frontend.rs) | Build LogicalPlans from SQL |
199+
| query | [`sql_ops/query.rs`](examples/sql_ops/query.rs) | Query data using SQL |
200200

201201
## UDF Examples
202202

datafusion-examples/examples/builtin_functions/main.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ async fn main() -> Result<()> {
8080

8181
let example: ExampleKind = std::env::args()
8282
.nth(1)
83-
.ok_or_else(|| DataFusionError::Execution(format!("Missing argument. {usage}")))?
83+
.unwrap_or_else(|| ExampleKind::All.to_string())
8484
.parse()
8585
.map_err(|_| DataFusionError::Execution(format!("Unknown example. {usage}")))?;
8686

datafusion-examples/examples/custom_data_source/main.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ async fn main() -> Result<()> {
108108

109109
let example: ExampleKind = std::env::args()
110110
.nth(1)
111-
.ok_or_else(|| DataFusionError::Execution(format!("Missing argument. {usage}")))?
111+
.unwrap_or_else(|| ExampleKind::All.to_string())
112112
.parse()
113113
.map_err(|_| DataFusionError::Execution(format!("Unknown example. {usage}")))?;
114114

datafusion-examples/examples/data_io/main.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ async fn main() -> Result<()> {
116116

117117
let example: ExampleKind = std::env::args()
118118
.nth(1)
119-
.ok_or_else(|| DataFusionError::Execution(format!("Missing argument. {usage}")))?
119+
.unwrap_or_else(|| ExampleKind::All.to_string())
120120
.parse()
121121
.map_err(|_| DataFusionError::Execution(format!("Unknown example. {usage}")))?;
122122

datafusion-examples/examples/dataframe/main.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ async fn main() -> Result<()> {
8585

8686
let example: ExampleKind = std::env::args()
8787
.nth(1)
88-
.ok_or_else(|| DataFusionError::Execution(format!("Missing argument. {usage}")))?
88+
.unwrap_or_else(|| ExampleKind::All.to_string())
8989
.parse()
9090
.map_err(|_| DataFusionError::Execution(format!("Unknown example. {usage}")))?;
9191

0 commit comments

Comments
 (0)