From 42c53b7d55a192ab77c8a6456bb0a88344e3c696 Mon Sep 17 00:00:00 2001 From: Shawn Chang Date: Tue, 17 Mar 2026 11:11:06 -0700 Subject: [PATCH 1/4] use resolving storage for python binding --- .../python/src/datafusion_table_provider.rs | 29 ++----------------- 1 file changed, 3 insertions(+), 26 deletions(-) diff --git a/bindings/python/src/datafusion_table_provider.rs b/bindings/python/src/datafusion_table_provider.rs index 7fa9f53dbd..7993ff0814 100644 --- a/bindings/python/src/datafusion_table_provider.rs +++ b/bindings/python/src/datafusion_table_provider.rs @@ -22,39 +22,16 @@ use std::sync::Arc; use datafusion_ffi::proto::logical_extension_codec::FFI_LogicalExtensionCodec; use datafusion_ffi::table_provider::FFI_TableProvider; use iceberg::TableIdent; -use iceberg::io::{FileIOBuilder, StorageFactory}; +use iceberg::io::FileIOBuilder; use iceberg::table::StaticTable; use iceberg_datafusion::table::IcebergStaticTableProvider; -use iceberg_storage_opendal::OpenDalStorageFactory; +use iceberg_storage_opendal::OpenDalResolvingStorageFactory; use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::{PyAnyMethods, PyCapsuleMethods, *}; use pyo3::types::{PyAny, PyCapsule}; use crate::runtime::runtime; -/// Parse the scheme from a URL and return the appropriate StorageFactory. -fn storage_factory_from_path(path: &str) -> PyResult> { - let scheme = path - .split("://") - .next() - .ok_or_else(|| PyRuntimeError::new_err(format!("Invalid path, missing scheme: {path}")))?; - - let factory: Arc = match scheme { - "file" | "" => Arc::new(OpenDalStorageFactory::Fs), - "s3" | "s3a" => Arc::new(OpenDalStorageFactory::S3 { - configured_scheme: scheme.to_string(), - customized_credential_load: None, - }), - "memory" => Arc::new(OpenDalStorageFactory::Memory), - _ => { - return Err(PyRuntimeError::new_err(format!( - "Unsupported storage scheme: {scheme}" - ))); - } - }; - - Ok(factory) -} pub(crate) fn validate_pycapsule(capsule: &Bound, name: &str) -> PyResult<()> { let capsule_name = capsule.name()?; @@ -110,7 +87,7 @@ impl PyIcebergDataFusionTable { let table_ident = TableIdent::from_strs(identifier) .map_err(|e| PyRuntimeError::new_err(format!("Invalid table identifier: {e}")))?; - let factory = storage_factory_from_path(&metadata_location)?; + let factory = Arc::new(OpenDalResolvingStorageFactory::new()); let mut builder = FileIOBuilder::new(factory); From 1817c98b22afd39ebcd351d4ddbf5e5c7f547396 Mon Sep 17 00:00:00 2001 From: Shawn Chang Date: Tue, 17 Mar 2026 11:19:34 -0700 Subject: [PATCH 2/4] fmt --- bindings/python/src/datafusion_table_provider.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/bindings/python/src/datafusion_table_provider.rs b/bindings/python/src/datafusion_table_provider.rs index 7993ff0814..95b3eb90d0 100644 --- a/bindings/python/src/datafusion_table_provider.rs +++ b/bindings/python/src/datafusion_table_provider.rs @@ -32,7 +32,6 @@ use pyo3::types::{PyAny, PyCapsule}; use crate::runtime::runtime; - pub(crate) fn validate_pycapsule(capsule: &Bound, name: &str) -> PyResult<()> { let capsule_name = capsule.name()?; if capsule_name.is_none() { From d2ff411e4ce05fbdcba1ed7c1aebe882493b6fc6 Mon Sep 17 00:00:00 2001 From: Shawn Chang Date: Tue, 17 Mar 2026 15:45:37 -0700 Subject: [PATCH 3/4] update toml to include all features --- bindings/python/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index 4813c72db7..0af4532194 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -33,7 +33,7 @@ crate-type = ["cdylib"] [dependencies] arrow = { version = "57.1", features = ["pyarrow", "chrono-tz"] } iceberg = { path = "../../crates/iceberg" } -iceberg-storage-opendal = { path = "../../crates/storage/opendal", features = ["opendal-s3", "opendal-fs", "opendal-memory"] } +iceberg-storage-opendal = { path = "../../crates/storage/opendal", features = ["opendal-memory", "opendal-fs", "opendal-s3", "opendal-gcs", "opendal-oss", "opendal-azdls"] } pyo3 = { version = "0.26", features = ["extension-module", "abi3-py310"] } iceberg-datafusion = { path = "../../crates/integrations/datafusion" } datafusion-ffi = { version = "52.1" } From ae3b505737a01ee3f3bd9aafa528350aeaca845d Mon Sep 17 00:00:00 2001 From: Shawn Chang Date: Wed, 18 Mar 2026 09:36:38 -0700 Subject: [PATCH 4/4] use opendal-all --- bindings/python/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index 0af4532194..9b551ea205 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -33,7 +33,7 @@ crate-type = ["cdylib"] [dependencies] arrow = { version = "57.1", features = ["pyarrow", "chrono-tz"] } iceberg = { path = "../../crates/iceberg" } -iceberg-storage-opendal = { path = "../../crates/storage/opendal", features = ["opendal-memory", "opendal-fs", "opendal-s3", "opendal-gcs", "opendal-oss", "opendal-azdls"] } +iceberg-storage-opendal = { path = "../../crates/storage/opendal", features = ["opendal-all"] } pyo3 = { version = "0.26", features = ["extension-module", "abi3-py310"] } iceberg-datafusion = { path = "../../crates/integrations/datafusion" } datafusion-ffi = { version = "52.1" }