refactor!(storage): Move OpenDal Storage to a new crate iceberg-storage-opendal

.github/workflows/publish.yml

@@ -39,6 +39,7 @@ jobs:
         # Order here is sensitive, as it will be used to determine the order of publishing
         package:
           - "crates/iceberg"
+          - "crates/storage/opendal"
           - "crates/catalog/glue"
           - "crates/catalog/hms"
           - "crates/catalog/rest"

Cargo.toml

@@ -24,6 +24,7 @@ members = [
   "crates/integration_tests",
   "crates/integrations/*",
   "crates/sqllogictest",
+  "crates/storage/*",
   "crates/test_utils",
 ]
 resolver = "2"
@@ -59,6 +60,7 @@ backon = "1.5.1"
 base64 = "0.22.1"
 bimap = "0.6"
 bytes = "1.11"
+cfg-if = "1"
 chrono = "0.4.41"
 clap = { version = "4.5.48", features = ["derive", "cargo"] }
 dashmap = "6"
@@ -85,6 +87,7 @@ iceberg-catalog-rest = { version = "0.8.0", path = "./crates/catalog/rest" }
 iceberg-catalog-s3tables = { version = "0.8.0", path = "./crates/catalog/s3tables" }
 iceberg-catalog-sql = { version = "0.8.0", path = "./crates/catalog/sql" }
 iceberg-datafusion = { version = "0.8.0", path = "./crates/integrations/datafusion" }
+iceberg-storage-opendal = { version = "0.8.0", path = "./crates/storage/opendal" }
 indicatif = "0.18"
 itertools = "0.13"
 libtest-mimic = "0.8.1"

bindings/python/Cargo.toml

@@ -33,6 +33,7 @@ crate-type = ["cdylib"]
 [dependencies]
 arrow = { version = "57.1", features = ["pyarrow", "chrono-tz"] }
 iceberg = { path = "../../crates/iceberg" }
+iceberg-storage-opendal = { path = "../../crates/storage/opendal", features = ["opendal-s3", "opendal-fs", "opendal-memory"] }
 pyo3 = { version = "0.26", features = ["extension-module", "abi3-py310"] }
 iceberg-datafusion = { path = "../../crates/integrations/datafusion" }
 datafusion-ffi = { version = "52.1" }

bindings/python/src/datafusion_table_provider.rs

@@ -22,9 +22,10 @@ use std::sync::Arc;
 use datafusion_ffi::proto::logical_extension_codec::FFI_LogicalExtensionCodec;
 use datafusion_ffi::table_provider::FFI_TableProvider;
 use iceberg::TableIdent;
-use iceberg::io::{FileIOBuilder, OpenDalStorageFactory, StorageFactory};
+use iceberg::io::{FileIOBuilder, StorageFactory};
 use iceberg::table::StaticTable;
 use iceberg_datafusion::table::IcebergStaticTableProvider;
+use iceberg_storage_opendal::OpenDalStorageFactory;
 use pyo3::exceptions::{PyRuntimeError, PyValueError};
 use pyo3::prelude::{PyAnyMethods, PyCapsuleMethods, *};
 use pyo3::types::{PyAny, PyCapsule};

crates/catalog/glue/Cargo.toml

@@ -34,6 +34,7 @@ async-trait = { workspace = true }
 aws-config = { workspace = true }
 aws-sdk-glue = { workspace = true }
 iceberg = { workspace = true }
+iceberg-storage-opendal = { workspace = true, features = ["opendal-s3"] }
 serde_json = { workspace = true }
 tokio = { workspace = true }
 tracing = { workspace = true }

crates/catalog/glue/src/catalog.rs

@@ -25,15 +25,16 @@ use aws_sdk_glue::operation::create_table::CreateTableError;
 use aws_sdk_glue::operation::update_table::UpdateTableError;
 use aws_sdk_glue::types::TableInput;
 use iceberg::io::{
-    FileIO, FileIOBuilder, OpenDalStorageFactory, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION,
-    S3_SECRET_ACCESS_KEY, S3_SESSION_TOKEN, StorageFactory,
+    FileIO, FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY,
+    S3_SESSION_TOKEN, StorageFactory,
 };
 use iceberg::spec::{TableMetadata, TableMetadataBuilder};
 use iceberg::table::Table;
 use iceberg::{
     Catalog, CatalogBuilder, Error, ErrorKind, MetadataLocation, Namespace, NamespaceIdent, Result,
     TableCommit, TableCreation, TableIdent,
 };
+use iceberg_storage_opendal::OpenDalStorageFactory;
 
 use crate::error::{from_aws_build_error, from_aws_sdk_error};
 use crate::utils::{

crates/catalog/glue/tests/glue_catalog_test.rs

@@ -23,10 +23,7 @@
 use std::collections::HashMap;
 use std::sync::Arc;
 
-use iceberg::io::{
-    FileIOBuilder, OpenDalStorageFactory, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION,
-    S3_SECRET_ACCESS_KEY,
-};
+use iceberg::io::{FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY};
 use iceberg::spec::{NestedField, PrimitiveType, Schema, Type};
 use iceberg::transaction::{ApplyTransactionAction, Transaction};
 use iceberg::{
@@ -36,6 +33,7 @@ use iceberg_catalog_glue::{
     AWS_ACCESS_KEY_ID, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, GLUE_CATALOG_PROP_URI,
     GLUE_CATALOG_PROP_WAREHOUSE, GlueCatalog, GlueCatalogBuilder,
 };
+use iceberg_storage_opendal::OpenDalStorageFactory;
 use iceberg_test_utils::{
     cleanup_namespace, get_glue_endpoint, get_minio_endpoint, normalize_test_name_with_parts,
     set_up,

crates/catalog/hms/Cargo.toml

@@ -54,6 +54,7 @@ motore-macros = { workspace = true }
 volo = { workspace = true }
 
 [dev-dependencies]
+iceberg-storage-opendal = { workspace = true, features = ["opendal-s3"] }
 iceberg_test_utils = { path = "../../test_utils", features = ["tests"] }
 
 [package.metadata.cargo-machete]

crates/catalog/hms/tests/hms_catalog_test.rs

@@ -23,16 +23,14 @@
 use std::collections::HashMap;
 use std::sync::Arc;
 
-use iceberg::io::{
-    FileIOBuilder, OpenDalStorageFactory, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION,
-    S3_SECRET_ACCESS_KEY,
-};
+use iceberg::io::{FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY};
 use iceberg::spec::{NestedField, PrimitiveType, Schema, Type};
 use iceberg::{Catalog, CatalogBuilder, Namespace, NamespaceIdent, TableCreation, TableIdent};
 use iceberg_catalog_hms::{
     HMS_CATALOG_PROP_THRIFT_TRANSPORT, HMS_CATALOG_PROP_URI, HMS_CATALOG_PROP_WAREHOUSE,
     HmsCatalog, HmsCatalogBuilder, THRIFT_TRANSPORT_BUFFERED,
 };
+use iceberg_storage_opendal::OpenDalStorageFactory;
 use iceberg_test_utils::{
     cleanup_namespace, get_hms_endpoint, get_minio_endpoint, normalize_test_name_with_parts, set_up,
 };

crates/catalog/s3tables/Cargo.toml

@@ -35,6 +35,7 @@ async-trait = { workspace = true }
 aws-config = { workspace = true }
 aws-sdk-s3tables = { workspace = true }
 iceberg = { workspace = true }
+iceberg-storage-opendal = { workspace = true, features = ["opendal-s3"] }
 
 
 [dev-dependencies]

crates/catalog/s3tables/src/catalog.rs

@@ -26,13 +26,14 @@ use aws_sdk_s3tables::operation::get_table::GetTableOutput;
 use aws_sdk_s3tables::operation::list_tables::ListTablesOutput;
 use aws_sdk_s3tables::operation::update_table_metadata_location::UpdateTableMetadataLocationError;
 use aws_sdk_s3tables::types::OpenTableFormat;
-use iceberg::io::{FileIO, FileIOBuilder, OpenDalStorageFactory, StorageFactory};
+use iceberg::io::{FileIO, FileIOBuilder, StorageFactory};
 use iceberg::spec::{TableMetadata, TableMetadataBuilder};
 use iceberg::table::Table;
 use iceberg::{
     Catalog, CatalogBuilder, Error, ErrorKind, MetadataLocation, Namespace, NamespaceIdent, Result,
     TableCommit, TableCreation, TableIdent,
 };
+use iceberg_storage_opendal::OpenDalStorageFactory;
 
 use crate::utils::create_sdk_config;
 

crates/examples/Cargo.toml

@@ -28,6 +28,7 @@ version = { workspace = true }
 futures = { workspace = true }
 iceberg = { workspace = true }
 iceberg-catalog-rest = { workspace = true }
+iceberg-storage-opendal = { workspace = true, optional = true }
 tokio = { workspace = true, features = ["full"] }
 
 [[example]]
@@ -45,4 +46,4 @@ required-features = ["storage-oss"]
 
 [features]
 default = []
-storage-oss = ["iceberg/storage-oss"]
+storage-oss = ["iceberg-storage-opendal/opendal-oss"]

crates/examples/src/oss_backend.rs

@@ -19,9 +19,9 @@ use std::collections::HashMap;
 use std::sync::Arc;
 
 use futures::stream::StreamExt;
-use iceberg::io::OpenDalStorageFactory;
 use iceberg::{Catalog, CatalogBuilder, NamespaceIdent, TableIdent};
 use iceberg_catalog_rest::{REST_CATALOG_PROP_URI, RestCatalogBuilder};
+use iceberg_storage_opendal::OpenDalStorageFactory;
 
 // Configure these values according to your environment
 

crates/iceberg/Cargo.toml

@@ -29,15 +29,7 @@ license = { workspace = true }
 repository = { workspace = true }
 
 [features]
-default = ["storage-memory", "storage-fs", "storage-s3"]
-storage-all = ["storage-memory", "storage-fs", "storage-s3", "storage-gcs"]
-
-storage-azdls = ["opendal/services-azdls"]
-storage-fs = ["opendal/services-fs"]
-storage-gcs = ["opendal/services-gcs"]
-storage-memory = ["opendal/services-memory"]
-storage-oss = ["opendal/services-oss"]
-storage-s3 = ["opendal/services-s3", "reqsign"]
+default = []
 
 
 [dependencies]
@@ -68,11 +60,9 @@ itertools = { workspace = true }
 moka = { version = "0.12.10", features = ["future"] }
 murmur3 = { workspace = true }
 once_cell = { workspace = true }
-opendal = { workspace = true }
 ordered-float = { workspace = true }
 parquet = { workspace = true, features = ["async"] }
 rand = { workspace = true }
-reqsign = { version = "0.16.3", optional = true, default-features = false }
 reqwest = { workspace = true }
 roaring = { workspace = true }
 fastnum = { workspace = true }

crates/iceberg/src/catalog/mod.rs

@@ -129,7 +129,8 @@ pub trait CatalogBuilder: Default + Debug + Send + Sync {
     ///
     /// ```rust,ignore
     /// use iceberg::CatalogBuilder;
-    /// use iceberg::io::{OpenDalStorageFactory, StorageFactory};
+    /// use iceberg::io::StorageFactory;
+    /// use iceberg_storage_opendal::OpenDalStorageFactory;
     /// use std::sync::Arc;
     ///
     /// let catalog = MyCatalogBuilder::default()

crates/iceberg/src/error.rs

@@ -384,12 +384,6 @@ define_from_err!(
     "Failure in conversion with avro"
 );
 
-define_from_err!(
-    opendal::Error,
-    ErrorKind::Unexpected,
-    "Failure in doing io operation"
-);
-
 define_from_err!(
     url::ParseError,
     ErrorKind::DataInvalid,

crates/iceberg/src/io/storage/mod.rs

@@ -20,7 +20,6 @@
 mod config;
 mod local_fs;
 mod memory;
-mod opendal;
 
 use std::fmt::Debug;
 use std::sync::Arc;
@@ -30,9 +29,6 @@ use bytes::Bytes;
 pub use config::*;
 pub use local_fs::{LocalFsStorage, LocalFsStorageFactory};
 pub use memory::{MemoryStorage, MemoryStorageFactory};
-#[cfg(feature = "storage-s3")]
-pub use opendal::CustomAwsCredentialLoader;
-pub use opendal::{OpenDalStorage, OpenDalStorageFactory};
 
 use super::{FileMetadata, FileRead, FileWrite, InputFile, OutputFile};
 use crate::Result;
@@ -69,11 +65,6 @@ use crate::Result;
 ///     }
 ///     // ... implement other methods
 /// }
-///
-/// TODO remove below when the trait is integrated with FileIO and Catalog
-/// # NOTE
-/// This trait is under heavy development and is not used anywhere as of now
-/// Please DO NOT implement it
 /// ```
 #[async_trait]
 #[typetag::serde(tag = "type")]
@@ -130,11 +121,6 @@ pub trait Storage: Debug + Send + Sync {
 ///         todo!()
 ///     }
 /// }
-///
-/// TODO remove below when the trait is integrated with FileIO and Catalog
-/// # NOTE
-/// This trait is under heavy development and is not used anywhere as of now
-/// Please DO NOT implement it
 /// ```
 #[typetag::serde(tag = "type")]
 pub trait StorageFactory: Debug + Send + Sync {

crates/integration_tests/Cargo.toml

@@ -30,6 +30,7 @@ arrow-schema = { workspace = true }
 futures = { workspace = true }
 iceberg = { workspace = true }
 iceberg-catalog-rest = { workspace = true }
+iceberg-storage-opendal = { workspace = true, features = ["opendal-s3"] }
 iceberg_test_utils = { path = "../test_utils", features = ["tests"] }
 parquet = { workspace = true }
 tokio = { workspace = true }

crates/integration_tests/tests/common/mod.rs

@@ -18,11 +18,11 @@
 use std::collections::HashMap;
 use std::sync::Arc;
 
-use iceberg::io::OpenDalStorageFactory;
 use iceberg::spec::{NestedField, PrimitiveType, Schema, Type};
 use iceberg::{Catalog, CatalogBuilder, Namespace, NamespaceIdent};
 use iceberg_catalog_rest::RestCatalogBuilder;
 use iceberg_integration_tests::get_test_fixture;
+use iceberg_storage_opendal::OpenDalStorageFactory;
 
 pub async fn random_ns() -> Namespace {
     let fixture = get_test_fixture();

crates/integration_tests/tests/conflict_commit_test.rs

@@ -24,7 +24,6 @@ use std::sync::Arc;
 use arrow_array::{ArrayRef, BooleanArray, Int32Array, RecordBatch, StringArray};
 use common::{random_ns, test_schema};
 use futures::TryStreamExt;
-use iceberg::io::OpenDalStorageFactory;
 use iceberg::transaction::{ApplyTransactionAction, Transaction};
 use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder;
 use iceberg::writer::file_writer::ParquetWriterBuilder;
@@ -36,6 +35,7 @@ use iceberg::writer::{IcebergWriter, IcebergWriterBuilder};
 use iceberg::{Catalog, CatalogBuilder, TableCreation};
 use iceberg_catalog_rest::RestCatalogBuilder;
 use iceberg_integration_tests::get_test_fixture;
+use iceberg_storage_opendal::OpenDalStorageFactory;
 use parquet::file::properties::WriterProperties;
 
 #[tokio::test]

crates/integration_tests/tests/read_evolved_schema.rs

@@ -22,11 +22,11 @@ use std::sync::Arc;
 use arrow_array::{Decimal128Array, Float64Array, Int64Array, StringArray};
 use futures::TryStreamExt;
 use iceberg::expr::Reference;
-use iceberg::io::OpenDalStorageFactory;
 use iceberg::spec::Datum;
 use iceberg::{Catalog, CatalogBuilder, TableIdent};
 use iceberg_catalog_rest::RestCatalogBuilder;
 use iceberg_integration_tests::get_test_fixture;
+use iceberg_storage_opendal::OpenDalStorageFactory;
 use ordered_float::OrderedFloat;
 
 #[tokio::test]