diff --git a/CMakeLists.txt b/CMakeLists.txt index da88eef..37880a0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,7 @@ include_directories(src/include) set(EXTENSION_SOURCES src/lance_extension.cpp src/lance_scan.cpp src/lance_search.cpp src/lance_common.cpp + src/lance_secrets.cpp src/lance_filter_ir.cpp src/lance_storage.cpp src/lance_metadata.cpp diff --git a/README.md b/README.md index 4c7569d..b9eb134 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,8 @@ duckdb -unsigned -c "LOAD 'build/release/extension/lance/lance.duckdb_extension' ## Usage -Full SQL reference: `docs/sql.md` +- Full SQL reference: [`docs/sql.md`](./docs/sql.md) +- Cloud storage reference: [`docs/cloud.md`](./docs/cloud.md) ### Query a Lance dataset @@ -58,10 +59,14 @@ SELECT * LIMIT 10; ``` -To read `s3://` paths, the extension can use DuckDB's native Secrets mechanism to obtain credentials: +To access object store URIs (e.g. `s3://...`), configure a `TYPE LANCE` secret (see [`docs/cloud.md`](./docs/cloud.md)). ```sql -CREATE SECRET (TYPE S3, provider credential_chain); +CREATE SECRET ( + TYPE LANCE, + PROVIDER credential_chain, + SCOPE 's3://bucket/' +); SELECT * FROM 's3://bucket/path/to/dataset.lance' @@ -95,13 +100,14 @@ COPY ( ) TO 'path/to/empty.lance' (FORMAT lance, mode 'overwrite', write_empty_file true); ``` -To write to `s3://...` paths, load DuckDB's `httpfs` extension and configure an S3 secret: +To write to `s3://...` paths, configure a `TYPE LANCE` secret for that scope (see [`docs/cloud.md`](./docs/cloud.md)). ```sql -INSTALL httpfs; -LOAD httpfs; - -CREATE SECRET (TYPE S3, provider credential_chain); +CREATE SECRET ( + TYPE LANCE, + PROVIDER credential_chain, + SCOPE 's3://bucket/' +); COPY (SELECT 1 AS id) TO 's3://bucket/path/to/out.lance' (FORMAT lance, mode 'overwrite'); ``` diff --git a/docs/cloud.md b/docs/cloud.md new file mode 100644 index 0000000..9eff707 --- /dev/null +++ b/docs/cloud.md @@ -0,0 +1,237 @@ +# Cloud Storage + +This extension can read and write Lance datasets stored in object stores by passing an object-store URI (e.g. `s3://...`) +to DuckDB, and configuring credentials and client settings via DuckDB's Secret Manager. + +This integration is intentionally thin: + +- You configure credentials and object store settings using `CREATE SECRET (TYPE LANCE, ...)`. +- The extension reads the matching secret by URI prefix (`SCOPE`) and forwards the key/value pairs to Lance as + `storage_options`. + +For the upstream Lance object store options and provider behavior, see https://lance.org/guide/object_store/. + +## Supported Backends + +Backends are supported if they are supported by the linked Lance build (see `Cargo.toml` features). + +Common URI schemes: + +- S3 / S3-compatible: `s3://...` (also accepts `s3a://...` and `s3n://...`, normalized to `s3://...`) +- Google Cloud Storage: `gs://...` +- Azure Blob Storage: `az://...` +- Alibaba Cloud OSS: `oss://...` +- Hugging Face Hub (OpenDAL): `hf://...` +- Local filesystem paths: `path/to/dataset.lance` + +## Secret Type: `LANCE` + +The extension registers a dedicated secret type with multiple providers: + +- `PROVIDER config`: Explicit key/value configuration in SQL. +- `PROVIDER credential_chain`: Create a scoped secret but rely on the upstream SDK credential chain (env vars, shared + config/credentials profiles, instance metadata) for credentials. +- `PROVIDER env`: Create a scoped secret but rely on environment variables in the upstream provider/SDK. + +All providers forward the resulting key/value pairs to Lance as `storage_options`. + +```sql +CREATE SECRET ( + TYPE LANCE, + PROVIDER config, + SCOPE 's3://my-bucket/', + ACCESS_KEY_ID '...', + SECRET_ACCESS_KEY '...', + REGION 'us-east-1' +); +``` + +### Scope Matching + +Secrets are matched using DuckDB's Secret Manager prefix matching: + +- `SCOPE` should be a URI prefix (e.g. `s3://bucket/`, `gs://bucket/`, `az://container/`). +- The best (longest) matching prefix is used when opening a dataset. + +### Parameter Binding and Forwarding + +DuckDB validates `CREATE SECRET` parameters at bind time. For `TYPE LANCE`, the extension maintains an allowlist of +known Lance storage option keys. Values are stored as strings and forwarded to Lance as-is. + +If you need to pass keys that are not in the allowlist yet, you can use the fallback `STORAGE_OPTIONS` map: + +```sql +CREATE SECRET ( + TYPE LANCE, + PROVIDER config, + SCOPE 's3://my-bucket/', + STORAGE_OPTIONS map(['my_custom_key'], ['my_custom_value']) +); +``` + +### Using the Credential Chain + +For object stores that have an upstream SDK credential chain (e.g. AWS), you can create a scoped secret without +embedding credentials in DuckDB: + +```sql +CREATE SECRET ( + TYPE LANCE, + PROVIDER credential_chain, + SCOPE 's3://my-bucket/', + REGION 'us-east-1' +); +``` + +## Common Options (Any Backend) + +These keys are commonly used across providers: + +- `ALLOW_HTTP`: Allow HTTP (insecure) connections (useful for MinIO / local testing). +- `ALLOW_INVALID_CERTIFICATES`: Allow invalid TLS certificates (use with care). +- `CONNECT_TIMEOUT`: Connection timeout (string value forwarded to Lance). +- `REQUEST_TIMEOUT`: Request timeout (string value forwarded to Lance). +- `CLIENT_MAX_RETRIES`: Retry count for object store client. +- `CLIENT_RETRY_TIMEOUT`: Retry timeout for object store client. +- `DOWNLOAD_RETRY_COUNT`: Download retry count for reads. +- `PROXY_URL`: Proxy URL. +- `PROXY_CA_CERTIFICATE`: Proxy CA certificate path or content (provider dependent). +- `PROXY_EXCLUDES`: Comma-separated proxy bypass list. +- `USER_AGENT`: Custom user agent. +- `EXPIRES_AT_MILLIS`: Optional expiration timestamp (milliseconds since epoch) for short-lived credentials. +- `USE_OPENDAL`: Use OpenDAL-based provider implementation when available (provider dependent). + +## S3 / S3-Compatible (`s3://`) + +Typical keys: + +- `ACCESS_KEY_ID`: AWS access key id. +- `SECRET_ACCESS_KEY`: AWS secret access key. +- `SESSION_TOKEN`: AWS session token (optional). +- `REGION`: AWS region. +- `ENDPOINT`: Custom endpoint (MinIO, localstack, non-AWS S3-compatible). +- `VIRTUAL_HOSTED_STYLE_REQUEST`: `true`/`false`. +- `SKIP_SIGNATURE`: `true` to use unsigned requests (anonymous / public buckets). + +The extension also accepts `AWS_*` aliases (e.g. `AWS_ACCESS_KEY_ID`, `AWS_REGION`, ...). + +Note: DuckDB requires `INSTALL httpfs;` and `LOAD httpfs;` before running +`ATTACH 's3://...' AS ... (TYPE LANCE, ...)`. + +Example (MinIO): + +```sql +CREATE SECRET ( + TYPE LANCE, + PROVIDER config, + SCOPE 's3://my-bucket/', + ACCESS_KEY_ID 'minioadmin', + SECRET_ACCESS_KEY 'minioadmin', + REGION 'us-east-1', + ENDPOINT 'http://127.0.0.1:9000', + VIRTUAL_HOSTED_STYLE_REQUEST false, + ALLOW_HTTP true +); +``` + +## Google Cloud Storage (`gs://`) + +Typical keys: + +- `SERVICE_ACCOUNT`: Service account email (OpenDAL, provider dependent). +- `SERVICE_ACCOUNT_KEY`: Service account key (JSON, provider dependent). +- `APPLICATION_CREDENTIALS`: Application credentials (provider dependent). +- `GOOGLE_APPLICATION_CREDENTIALS`: Path to credentials file (provider dependent). +- `GOOGLE_SERVICE_ACCOUNT`: Service account email (provider dependent). +- `GOOGLE_SERVICE_ACCOUNT_KEY`: Service account key (provider dependent). +- `GOOGLE_STORAGE_TOKEN`: Bearer token for GCS requests (when using object_store's GCS client). +- `USE_OPENDAL`: `true`/`false` to select OpenDAL implementation. + +Example (token-based): + +```sql +CREATE SECRET ( + TYPE LANCE, + PROVIDER config, + SCOPE 'gs://my-bucket/', + GOOGLE_STORAGE_TOKEN '...', + USE_OPENDAL false +); +``` + +## Azure Blob Storage (`az://`) + +Typical keys: + +- `ACCOUNT_NAME`: Storage account name. +- `ACCOUNT_KEY`: Storage account key (base64). +- `SAS_KEY` / `SAS_TOKEN`: SAS credentials (provider dependent). +- `BEARER_TOKEN`: Bearer token (provider dependent). +- `AZURE_*`: Additional Azure configuration keys forwarded to Lance (see Lance docs). +- `USE_OPENDAL`: `true`/`false` to select OpenDAL implementation. + +Note: Depending on the URI form, the account name may need to be present in either the URI authority or in the options. + +Example (account key): + +```sql +CREATE SECRET ( + TYPE LANCE, + PROVIDER config, + SCOPE 'az://my-container/', + ACCOUNT_NAME 'my-account', + ACCOUNT_KEY '...', + USE_OPENDAL true +); +``` + +## Alibaba Cloud OSS (`oss://`) + +Typical keys: + +- `OSS_ENDPOINT`: Required OSS endpoint. +- `OSS_ACCESS_KEY_ID`: Access key id. +- `OSS_SECRET_ACCESS_KEY`: Access key secret. +- `OSS_REGION`: Region (optional, provider dependent). + +Example: + +```sql +CREATE SECRET ( + TYPE LANCE, + PROVIDER config, + SCOPE 'oss://my-bucket/', + OSS_ENDPOINT 'https://oss-cn-hangzhou.aliyuncs.com', + OSS_ACCESS_KEY_ID '...', + OSS_SECRET_ACCESS_KEY '...' +); +``` + +## Hugging Face Hub (`hf://`) + +Hugging Face support is implemented via OpenDAL. URLs look like: + +- `hf://datasets///` + +Typical keys: + +- `HF_TOKEN`: Access token. +- `HF_REVISION`: Revision name (e.g. `main`, tag, commit). +- `HF_ROOT`: Root prefix inside the repo. + +Example: + +```sql +CREATE SECRET ( + TYPE LANCE, + PROVIDER config, + SCOPE 'hf://datasets/acme/my-repo/', + HF_TOKEN '...', + HF_REVISION 'main' +); +``` + +## Notes on Secret Redaction + +When listing secrets, DuckDB may redact values depending on secret display mode. Additionally, the `TYPE LANCE` secret +marks keys containing `secret`, `password`, or `token` as redacted by default. diff --git a/src/include/lance_common.hpp b/src/include/lance_common.hpp index fc4a97a..38b1e4e 100644 --- a/src/include/lance_common.hpp +++ b/src/include/lance_common.hpp @@ -15,10 +15,10 @@ void ApplyDuckDBFilters(ClientContext &context, TableFilterSet &filters, void *LanceOpenDataset(ClientContext &context, const string &path); string LanceNormalizeS3Scheme(const string &path); -void LanceFillS3StorageOptionsFromSecrets(ClientContext &context, - const string &path, - vector &out_keys, - vector &out_values); +void LanceFillStorageOptionsFromSecrets(ClientContext &context, + const string &path, + vector &out_keys, + vector &out_values); void ResolveLanceStorageOptions(ClientContext &context, const string &path, string &out_open_path, vector &out_option_keys, diff --git a/src/include/lance_secrets.hpp b/src/include/lance_secrets.hpp new file mode 100644 index 0000000..37ada65 --- /dev/null +++ b/src/include/lance_secrets.hpp @@ -0,0 +1,9 @@ +#pragma once + +#include "duckdb.hpp" + +namespace duckdb { + +void RegisterLanceSecrets(DatabaseInstance &instance); + +} // namespace duckdb diff --git a/src/lance_common.cpp b/src/lance_common.cpp index 217e0e7..5115652 100644 --- a/src/lance_common.cpp +++ b/src/lance_common.cpp @@ -62,22 +62,13 @@ static string SecretValueToString(const Value &value) { return value.ToString(); } -static void AddIfNotEmpty(vector &keys, vector &values, - const string &key, const string &value) { - if (value.empty()) { - return; - } - keys.push_back(key); - values.push_back(value); -} - -void LanceFillS3StorageOptionsFromSecrets(ClientContext &context, - const string &path, - vector &out_keys, - vector &out_values) { +void LanceFillStorageOptionsFromSecrets(ClientContext &context, + const string &path, + vector &out_keys, + vector &out_values) { auto &secret_manager = SecretManager::Get(context); auto transaction = CatalogTransaction::GetSystemCatalogTransaction(context); - auto secret_match = secret_manager.LookupSecret(transaction, path, "s3"); + auto secret_match = secret_manager.LookupSecret(transaction, path, "lance"); if (!secret_match.HasMatch() || !secret_match.secret_entry || !secret_match.secret_entry->secret) { return; @@ -89,40 +80,12 @@ void LanceFillS3StorageOptionsFromSecrets(ClientContext &context, return; } - auto key_id = SecretValueToString(kv_secret->TryGetValue("key_id")); - auto secret_access_key = - SecretValueToString(kv_secret->TryGetValue("secret")); - auto session_token = - SecretValueToString(kv_secret->TryGetValue("session_token")); - auto region = SecretValueToString(kv_secret->TryGetValue("region")); - auto endpoint = SecretValueToString(kv_secret->TryGetValue("endpoint")); - auto url_style = SecretValueToString(kv_secret->TryGetValue("url_style")); - auto use_ssl = SecretValueToString(kv_secret->TryGetValue("use_ssl")); - - if (key_id.empty() && secret_access_key.empty()) { - AddIfNotEmpty(out_keys, out_values, "skip_signature", "true"); - } else { - AddIfNotEmpty(out_keys, out_values, "access_key_id", key_id); - AddIfNotEmpty(out_keys, out_values, "secret_access_key", secret_access_key); - AddIfNotEmpty(out_keys, out_values, "session_token", session_token); - } - - AddIfNotEmpty(out_keys, out_values, "region", region); - AddIfNotEmpty(out_keys, out_values, "endpoint", endpoint); - - if (StringUtil::CIEquals(url_style, "vhost") || - StringUtil::CIEquals(url_style, "virtual_hosted")) { - AddIfNotEmpty(out_keys, out_values, "virtual_hosted_style_request", "true"); - } else if (StringUtil::CIEquals(url_style, "path")) { - AddIfNotEmpty(out_keys, out_values, "virtual_hosted_style_request", - "false"); - } - - if (!use_ssl.empty()) { - if (StringUtil::CIEquals(use_ssl, "false") || - StringUtil::CIEquals(use_ssl, "0")) { - AddIfNotEmpty(out_keys, out_values, "allow_http", "true"); + for (auto &kv : kv_secret->secret_map) { + if (kv.second.IsNull()) { + continue; } + out_keys.push_back(kv.first); + out_values.push_back(kv.second.ToString()); } } @@ -220,10 +183,8 @@ void ResolveLanceStorageOptions(ClientContext &context, const string &path, out_values.clear(); out_open_path = LanceNormalizeS3Scheme(out_open_path); - if (StringUtil::StartsWith(out_open_path, "s3://")) { - LanceFillS3StorageOptionsFromSecrets(context, out_open_path, out_keys, - out_values); - } + LanceFillStorageOptionsFromSecrets(context, out_open_path, out_keys, + out_values); } void BuildStorageOptionPointerArrays(const vector &option_keys, diff --git a/src/lance_extension.cpp b/src/lance_extension.cpp index 72c52c4..7ca43fa 100644 --- a/src/lance_extension.cpp +++ b/src/lance_extension.cpp @@ -1,6 +1,7 @@ #define DUCKDB_EXTENSION_MAIN #include "lance_extension.hpp" +#include "lance_secrets.hpp" #include "duckdb.hpp" #include "duckdb/common/exception.hpp" #include "duckdb/common/string_util.hpp" @@ -32,6 +33,7 @@ void LanceExtension::Load(ExtensionLoader &loader) { // Enable SELECT * FROM '.../dataset.lance' auto &instance = loader.GetDatabaseInstance(); + RegisterLanceSecrets(instance); auto &config = DBConfig::GetConfig(instance); RegisterLanceScanOptimizer(config); RegisterLanceStorage(config); diff --git a/src/lance_secrets.cpp b/src/lance_secrets.cpp new file mode 100644 index 0000000..c916bbd --- /dev/null +++ b/src/lance_secrets.cpp @@ -0,0 +1,179 @@ +#include "lance_secrets.hpp" + +#include "duckdb/common/string_util.hpp" +#include "duckdb/main/secret/secret_manager.hpp" + +namespace duckdb { + +static bool ShouldRedactLanceStorageOption(const string &key) { + return StringUtil::Contains(StringUtil::Lower(key), "secret") || + StringUtil::Contains(StringUtil::Lower(key), "password") || + StringUtil::Contains(StringUtil::Lower(key), "token"); +} + +static unique_ptr +CreateLanceSecretFromConfig(ClientContext &, CreateSecretInput &input) { + auto secret = make_uniq(input.scope, input.type, + input.provider, input.name); + + for (auto &kv : input.options) { + if (StringUtil::CIEquals(kv.first, "storage_options")) { + continue; + } + if (kv.second.IsNull()) { + continue; + } + secret->secret_map[kv.first] = Value(kv.second.ToString()); + } + + auto it = input.options.find("storage_options"); + if (it != input.options.end() && !it->second.IsNull()) { + auto &children = MapValue::GetChildren(it->second); + for (auto &child : children) { + auto &child_struct = StructValue::GetChildren(child); + if (child_struct[0].IsNull() || child_struct[1].IsNull()) { + continue; + } + auto key = child_struct[0].ToString(); + auto value = child_struct[1].ToString(); + if (key.empty()) { + continue; + } + secret->secret_map[key] = Value(std::move(value)); + } + } + + for (auto &kv : secret->secret_map) { + if (ShouldRedactLanceStorageOption(kv.first)) { + secret->redact_keys.insert(kv.first); + } + } + + return std::move(secret); +} + +static bool HasSecretType(SecretManager &secret_manager, const string &name) { + for (auto &t : secret_manager.AllSecretTypes()) { + if (StringUtil::CIEquals(t.name, name)) { + return true; + } + } + return false; +} + +void RegisterLanceSecrets(DatabaseInstance &instance) { + auto &secret_manager = SecretManager::Get(instance); + + if (!HasSecretType(secret_manager, "lance")) { + SecretType secret_type; + secret_type.name = "lance"; + secret_type.deserializer = KeyValueSecret::Deserialize; + secret_type.default_provider = "config"; + secret_type.extension = "lance"; + secret_manager.RegisterSecretType(secret_type); + } + + const auto storage_option_keys = vector{ + "access_key_id", + "secret_access_key", + "session_token", + "region", + "endpoint", + "virtual_hosted_style_request", + "skip_signature", + "allow_http", + "allow_invalid_certificates", + "connect_timeout", + "request_timeout", + "client_max_retries", + "client_retry_timeout", + "download_retry_count", + "proxy_url", + "proxy_ca_certificate", + "proxy_excludes", + "user_agent", + "expires_at_millis", + "use_opendal", + "s3_express", + "aws_access_key_id", + "aws_secret_access_key", + "aws_session_token", + "aws_region", + "aws_endpoint", + "aws_virtual_hosted_style_request", + "aws_server_side_encryption", + "aws_sse_kms_key_id", + "aws_sse_bucket_key_enabled", + "aws_s3_express", + "google_storage_token", + "service_account", + "service_account_key", + "application_credentials", + "google_application_credentials", + "google_service_account", + "google_service_account_key", + "account_name", + "account_key", + "sas_key", + "sas_token", + "bearer_token", + "access_key", + "azure_storage_account_name", + "azure_storage_account_key", + "azure_client_id", + "azure_client_secret", + "azure_tenant_id", + "azure_storage_sas_key", + "azure_storage_sas_token", + "azure_storage_token", + "azure_endpoint", + "azure_identity_endpoint", + "azure_msi_endpoint", + "azure_msi_resource_id", + "azure_object_id", + "azure_federated_token_file", + "azure_use_azure_cli", + "azure_use_fabric_endpoint", + "azure_disable_tagging", + "azure_storage_use_emulator", + "oss_endpoint", + "oss_access_key_id", + "oss_secret_access_key", + "oss_region", + "hf_token", + "hf_revision", + "hf_root", + }; + + // DuckDB's CREATE SECRET binder requires an explicit allowlist of named + // parameters. Values are stored as strings and forwarded to Lance as-is. + auto register_provider = [&](const string &provider) { + CreateSecretFunction fun; + fun.secret_type = "lance"; + fun.provider = provider; + fun.function = CreateLanceSecretFromConfig; + + for (auto &key : storage_option_keys) { + fun.named_parameters[key] = LogicalType::VARCHAR; + } + + // Fallback: accept a MAP for bulk configuration. + fun.named_parameters["storage_options"] = + LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR); + + secret_manager.RegisterSecretFunction( + std::move(fun), OnCreateConflict::REPLACE_ON_CONFLICT); + }; + + // Explicit key/value configuration. + register_provider("config"); + + // Semantic aliases: + // - "credential_chain": configure scope/overrides, but rely on the upstream + // SDK credential chain (env/profile/metadata services) for credentials. + // - "env": configure via environment variables in the upstream SDK/provider. + register_provider("credential_chain"); + register_provider("env"); +} + +} // namespace duckdb diff --git a/test/sql/README.md b/test/sql/README.md index bd7310c..ad8713b 100644 --- a/test/sql/README.md +++ b/test/sql/README.md @@ -13,7 +13,7 @@ Tests are grouped by prefix to keep related coverage discoverable: - `optimizer_*`: Optimizer/statistics integration - `namespace_*`: `ATTACH ... (TYPE LANCE)` namespace mapping and table discovery - `dml_*`: Write-path SQL (COPY/INSERT/UPDATE/DELETE/TRUNCATE/DROP/ALTER) -- `s3_*`: End-to-end S3 tests (MinIO), gated by `require-env` +- `s3_*`: End-to-end S3 tests (MinIO), gated by `require-env` and using a `TYPE LANCE` secret - `tpch.test`: TPC-H correctness suite - `bench_*`: Larger correctness suites and fixtures (e.g., BigANN) diff --git a/test/sql/s3_drop_table_minio.test b/test/sql/s3_drop_table_minio.test index ee69f80..7850871 100644 --- a/test/sql/s3_drop_table_minio.test +++ b/test/sql/s3_drop_table_minio.test @@ -1,5 +1,5 @@ # name: test/sql/s3_drop_table_minio.test -# description: DROP TABLE deletes Lance datasets in S3 directory namespaces (MinIO) +# description: DROP TABLE deletes Lance datasets in S3 directory namespaces (MinIO, TYPE LANCE secret) # group: [sql] require-env LANCE_TEST_S3 1 @@ -26,15 +26,15 @@ LOAD httpfs; statement ok CREATE SECRET ( - TYPE S3, + TYPE LANCE, PROVIDER config, SCOPE 's3://${LANCE_S3_BUCKET}/', - KEY_ID '${LANCE_S3_ACCESS_KEY_ID}', - SECRET '${LANCE_S3_SECRET_ACCESS_KEY}', + ACCESS_KEY_ID '${LANCE_S3_ACCESS_KEY_ID}', + SECRET_ACCESS_KEY '${LANCE_S3_SECRET_ACCESS_KEY}', REGION '${LANCE_S3_REGION}', ENDPOINT '${LANCE_S3_ENDPOINT}', - URL_STYLE 'path', - USE_SSL false + VIRTUAL_HOSTED_STYLE_REQUEST false, + ALLOW_HTTP true ); statement ok diff --git a/test/sql/s3_scan_minio.test b/test/sql/s3_scan_minio.test index 15d7179..3c5a763 100644 --- a/test/sql/s3_scan_minio.test +++ b/test/sql/s3_scan_minio.test @@ -1,5 +1,5 @@ # name: test/sql/s3_scan_minio.test -# description: End-to-end S3 scan against MinIO using DuckDB secrets +# description: End-to-end S3 scan against MinIO using DuckDB secrets (TYPE LANCE) # group: [sql] require-env LANCE_TEST_S3 1 @@ -20,23 +20,17 @@ test-env LANCE_S3_SECRET_ACCESS_KEY minioadmin require lance -statement ok -INSTALL httpfs; - -statement ok -LOAD httpfs; - statement ok CREATE SECRET ( - TYPE S3, + TYPE LANCE, PROVIDER config, SCOPE 's3://${LANCE_S3_BUCKET}/', - KEY_ID '${LANCE_S3_ACCESS_KEY_ID}', - SECRET '${LANCE_S3_SECRET_ACCESS_KEY}', + ACCESS_KEY_ID '${LANCE_S3_ACCESS_KEY_ID}', + SECRET_ACCESS_KEY '${LANCE_S3_SECRET_ACCESS_KEY}', REGION '${LANCE_S3_REGION}', ENDPOINT '${LANCE_S3_ENDPOINT}', - URL_STYLE 'path', - USE_SSL false + VIRTUAL_HOSTED_STYLE_REQUEST false, + ALLOW_HTTP true ); query I diff --git a/test/sql/s3_write_minio.test b/test/sql/s3_write_minio.test index a1975ce..cf7a958 100644 --- a/test/sql/s3_write_minio.test +++ b/test/sql/s3_write_minio.test @@ -1,5 +1,5 @@ # name: test/sql/s3_write_minio.test -# description: End-to-end S3 write against MinIO using DuckDB secrets +# description: End-to-end S3 write against MinIO using DuckDB secrets (TYPE LANCE) # group: [sql] require-env LANCE_TEST_S3 1 @@ -18,23 +18,17 @@ test-env LANCE_S3_WRITE_DATASET_PREFIX write_test_data.lance require lance -statement ok -INSTALL httpfs; - -statement ok -LOAD httpfs; - statement ok CREATE SECRET ( - TYPE S3, + TYPE LANCE, PROVIDER config, SCOPE 's3://${LANCE_S3_BUCKET}/', - KEY_ID '${LANCE_S3_ACCESS_KEY_ID}', - SECRET '${LANCE_S3_SECRET_ACCESS_KEY}', + ACCESS_KEY_ID '${LANCE_S3_ACCESS_KEY_ID}', + SECRET_ACCESS_KEY '${LANCE_S3_SECRET_ACCESS_KEY}', REGION '${LANCE_S3_REGION}', ENDPOINT '${LANCE_S3_ENDPOINT}', - URL_STYLE 'path', - USE_SSL false + VIRTUAL_HOSTED_STYLE_REQUEST false, + ALLOW_HTTP true ); statement ok