Skip to content

Commit 68506cd

Browse files
committed
NIFI-15062 Support configuring S3 endpoint/path style acces in Iceberg
1 parent 4f99d4a commit 68506cd

1 file changed

Lines changed: 28 additions & 1 deletion

File tree

nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-aws/src/main/java/org/apache/nifi/services/iceberg/aws/S3IcebergFileIOProvider.java

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,12 +102,32 @@ public class S3IcebergFileIOProvider extends AbstractControllerService implement
102102
)
103103
.build();
104104

105+
static final PropertyDescriptor ENDPOINT_OVERRIDE = new PropertyDescriptor.Builder()
106+
.name("Endpoint Override URL")
107+
.description("Endpoint URL to use instead of the AWS default including scheme, host, port, and path. " +
108+
"The AWS libraries select an endpoint URL based on the AWS region, but this property overrides " +
109+
"the selected endpoint URL, allowing use with other S3-compatible endpoints.")
110+
.expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT)
111+
.required(false)
112+
.addValidator(StandardValidators.URL_VALIDATOR)
113+
.build();
114+
115+
static final PropertyDescriptor USE_PATH_STYLE_ACCESS = new PropertyDescriptor.Builder()
116+
.name("Use Path Style Access")
117+
.description("Path-style access can be enforced by setting this property to true. Set it to true if your endpoint does not support " +
118+
"virtual-hosted-style requests, only path-style requests.")
119+
.allowableValues("true", "false")
120+
.defaultValue(String.valueOf(S3FileIOProperties.PATH_STYLE_ACCESS_DEFAULT))
121+
.build();
122+
105123
private static final List<PropertyDescriptor> PROPERTY_DESCRIPTORS = List.of(
106124
AUTHENTICATION_STRATEGY,
107125
ACCESS_KEY_ID,
108126
SECRET_ACCESS_KEY,
109127
SESSION_TOKEN,
110-
CLIENT_REGION
128+
CLIENT_REGION,
129+
ENDPOINT_OVERRIDE,
130+
USE_PATH_STYLE_ACCESS
111131
);
112132

113133
private final Map<String, String> standardProperties = new ConcurrentHashMap<>();
@@ -159,6 +179,13 @@ private Map<String, String> getConfiguredProperties(final ConfigurationContext c
159179
}
160180
}
161181

182+
if (context.getProperty(ENDPOINT_OVERRIDE).isSet()) {
183+
final String endpoint = context.getProperty(ENDPOINT_OVERRIDE).evaluateAttributeExpressions().getValue();
184+
contextProperties.put(S3FileIOProperties.ENDPOINT, endpoint);
185+
}
186+
final String pathStyleAccess = context.getProperty(USE_PATH_STYLE_ACCESS).getValue();
187+
contextProperties.put(S3FileIOProperties.PATH_STYLE_ACCESS, pathStyleAccess);
188+
162189
// HttpURLConnection Client Type avoids additional dependencies
163190
contextProperties.put(HttpClientProperties.CLIENT_TYPE, HttpClientProperties.CLIENT_TYPE_URLCONNECTION);
164191

0 commit comments

Comments
 (0)