diff --git a/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/credentials/CredentialsProvider.java b/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/credentials/CredentialsProvider.java index 787efb8f..237cb357 100644 --- a/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/credentials/CredentialsProvider.java +++ b/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/credentials/CredentialsProvider.java @@ -15,7 +15,6 @@ import java.util.Optional; -// TODO: Add back file-based provider public interface CredentialsProvider { CredentialsProvider NOOP = (_, _) -> Optional.empty(); diff --git a/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/remote/RemoteS3ConnectionProvider.java b/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/remote/RemoteS3ConnectionProvider.java index 705a5dc0..da297c07 100644 --- a/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/remote/RemoteS3ConnectionProvider.java +++ b/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/remote/RemoteS3ConnectionProvider.java @@ -24,5 +24,5 @@ public interface RemoteS3ConnectionProvider { RemoteS3ConnectionProvider NOOP = (_, _, _) -> Optional.empty(); - Optional remoteConnection(SigningMetadata signingMetadata, Optional identity, ParsedS3Request request); + Optional remoteConnection(SigningMetadata signingMetadata, Optional identity, ParsedS3Request request); } diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/TrinoAwsProxyServerModule.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/TrinoAwsProxyServerModule.java index 0f34a562..bbda3a77 100644 --- a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/TrinoAwsProxyServerModule.java +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/TrinoAwsProxyServerModule.java @@ -34,6 +34,9 @@ import io.trino.aws.proxy.server.credentials.http.HttpCredentialsModule; import io.trino.aws.proxy.server.remote.DefaultRemoteS3Module; import io.trino.aws.proxy.server.remote.RemoteS3ConnectionController; +import io.trino.aws.proxy.server.remote.provider.file.FileBasedRemoteS3ConnectionModule; +import io.trino.aws.proxy.server.remote.provider.http.HttpRemoteS3ConnectionProviderModule; +import io.trino.aws.proxy.server.remote.provider.preset.StaticRemoteS3ConnectionProviderModule; import io.trino.aws.proxy.server.rest.LimitStreamController; import io.trino.aws.proxy.server.rest.ResourceSecurityDynamicFeature; import io.trino.aws.proxy.server.rest.RestModule; @@ -134,6 +137,9 @@ protected void setup(Binder binder) install(new FileBasedCredentialsModule()); install(new OpaS3SecurityModule()); install(new HttpCredentialsModule()); + install(new FileBasedRemoteS3ConnectionModule()); + install(new StaticRemoteS3ConnectionProviderModule()); + install(new HttpRemoteS3ConnectionProviderModule()); configBinder(binder).bindConfig(RemoteS3Config.class); // RemoteS3 provided implementation diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/SerializableRemoteS3Connection.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/SerializableRemoteS3Connection.java new file mode 100644 index 00000000..ca8fda85 --- /dev/null +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/SerializableRemoteS3Connection.java @@ -0,0 +1,73 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.remote.provider; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.inject.ConfigurationException; +import io.airlift.configuration.ConfigurationFactory; +import io.trino.aws.proxy.server.remote.DefaultRemoteS3Config; +import io.trino.aws.proxy.server.remote.PathStyleRemoteS3Facade; +import io.trino.aws.proxy.server.remote.VirtualHostStyleRemoteS3Facade; +import io.trino.aws.proxy.spi.credentials.Credential; +import io.trino.aws.proxy.spi.remote.RemoteS3Connection; +import io.trino.aws.proxy.spi.remote.RemoteS3Facade; +import io.trino.aws.proxy.spi.remote.RemoteSessionRole; + +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static com.google.common.collect.Sets.difference; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +public record SerializableRemoteS3Connection( + Credential remoteCredential, + Optional remoteSessionRole, + Optional remoteS3Facade) + implements RemoteS3Connection +{ + public SerializableRemoteS3Connection + { + requireNonNull(remoteCredential, "remoteCredential is null"); + requireNonNull(remoteSessionRole, "remoteSessionRole is null"); + requireNonNull(remoteS3Facade, "remoteS3Facade is null"); + } + + @JsonCreator + public static SerializableRemoteS3Connection fromConfig( + @JsonProperty("remoteCredential") Credential remoteCredential, + @JsonProperty("remoteSessionRole") Optional remoteSessionRole, + @JsonProperty("remoteS3FacadeConfiguration") Optional> remoteS3FacadeConfiguration) + { + Optional facade = remoteS3FacadeConfiguration.map(config -> { + ConfigurationFactory configurationFactory = new ConfigurationFactory(config); + DefaultRemoteS3Config parsedConfig; + try { + parsedConfig = configurationFactory.build(DefaultRemoteS3Config.class); + } + catch (ConfigurationException e) { + throw new IllegalArgumentException("Failed create RemoteS3Facade from RemoteS3FacadeConfiguration", e); + } + Set unusedProperties = difference(configurationFactory.getProperties().keySet(), configurationFactory.getUsedProperties()); + if (!unusedProperties.isEmpty()) { + throw new IllegalArgumentException(format("Failed to create RemoteS3Facade from RemoteS3FacadeConfiguration. Unused properties when instantiating " + + "DefaultRemoteS3Config: %s", unusedProperties)); + } + return parsedConfig.getVirtualHostStyle() ? new VirtualHostStyleRemoteS3Facade(parsedConfig) : new PathStyleRemoteS3Facade(parsedConfig); + }); + return new SerializableRemoteS3Connection(remoteCredential, remoteSessionRole, facade); + } +} diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/file/FileBasedRemoteS3ConnectionModule.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/file/FileBasedRemoteS3ConnectionModule.java new file mode 100644 index 00000000..24b5bc34 --- /dev/null +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/file/FileBasedRemoteS3ConnectionModule.java @@ -0,0 +1,42 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.remote.provider.file; + +import com.google.inject.Binder; +import io.airlift.configuration.AbstractConfigurationAwareModule; +import io.trino.aws.proxy.server.remote.provider.SerializableRemoteS3Connection; + +import static io.airlift.configuration.ConfigBinder.configBinder; +import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; +import static io.trino.aws.proxy.spi.plugin.TrinoAwsProxyServerBinding.remoteS3ConnectionProviderModule; + +public class FileBasedRemoteS3ConnectionModule + extends AbstractConfigurationAwareModule +{ + // set as config value for "remote-s3-connection-provider.type" + public static final String FILE_BASED_REMOTE_S3_CONNECTION_PROVIDER = "file"; + + @Override + protected void setup(Binder binder) + { + install(remoteS3ConnectionProviderModule( + FILE_BASED_REMOTE_S3_CONNECTION_PROVIDER, + FileBasedRemoteS3ConnectionProvider.class, + innerBinder -> { + configBinder(innerBinder).bindConfig(FileBasedRemoteS3ConnectionProviderConfig.class); + innerBinder.bind(FileBasedRemoteS3ConnectionProvider.class); + jsonCodecBinder(innerBinder).bindMapJsonCodec(String.class, SerializableRemoteS3Connection.class); + })); + } +} diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/file/FileBasedRemoteS3ConnectionProvider.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/file/FileBasedRemoteS3ConnectionProvider.java new file mode 100644 index 00000000..35c48f41 --- /dev/null +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/file/FileBasedRemoteS3ConnectionProvider.java @@ -0,0 +1,77 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.remote.provider.file; + +import com.google.common.io.Files; +import com.google.inject.Inject; +import io.airlift.json.JsonCodec; +import io.trino.aws.proxy.server.remote.provider.SerializableRemoteS3Connection; +import io.trino.aws.proxy.spi.credentials.Identity; +import io.trino.aws.proxy.spi.remote.RemoteS3Connection; +import io.trino.aws.proxy.spi.remote.RemoteS3ConnectionProvider; +import io.trino.aws.proxy.spi.rest.ParsedS3Request; +import io.trino.aws.proxy.spi.signing.SigningMetadata; + +import java.util.Map; +import java.util.Optional; + +/** + *

File-based RemoteS3ConnectionProvider that reads a JSON file containing a mapping from emulated access key to + * RemoteS3Connection.

+ * 
{@code
+ * {
+ *   "emulated-access-key-1": {
+ *     "remoteCredential": {
+ *       "accessKey": "remote-access-key",
+ *       "secretKey": "remote-secret-key"
+ *     },
+ *     "remoteSessionRole": {
+ *       "region": "us-east-1",
+ *       "roleArn": "arn:aws:iam::123456789012:role/role-name",
+ *       "externalId": "external-id",
+ *       "stsEndpoint": "https://sts.us-east-1.amazonaws.com"
+ *     },
+ *     "remoteS3FacadeConfiguration": {
+ *       "remoteS3.https": true,
+ *       "remoteS3.domain": "s3.amazonaws.com",
+ *       "remoteS3.port": 443,
+ *       "remoteS3.virtual-host-style": false,
+ *       "remoteS3.hostname.template": "${domain}"
+ *     }
+ *   }
+ * }
+ * }
+ */ +public class FileBasedRemoteS3ConnectionProvider + implements RemoteS3ConnectionProvider +{ + private final Map remoteS3Connections; + + @Inject + public FileBasedRemoteS3ConnectionProvider(FileBasedRemoteS3ConnectionProviderConfig config, JsonCodec> jsonCodec) + { + try { + this.remoteS3Connections = jsonCodec.fromJson(Files.toByteArray(config.getConnectionsFile())); + } + catch (Exception e) { + throw new RuntimeException("Failed to read remote S3 connections file", e); + } + } + + @Override + public Optional remoteConnection(SigningMetadata signingMetadata, Optional identity, ParsedS3Request request) + { + return Optional.ofNullable(remoteS3Connections.get(signingMetadata.credential().accessKey())); + } +} diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/file/FileBasedRemoteS3ConnectionProviderConfig.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/file/FileBasedRemoteS3ConnectionProviderConfig.java new file mode 100644 index 00000000..9bdac2b5 --- /dev/null +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/file/FileBasedRemoteS3ConnectionProviderConfig.java @@ -0,0 +1,39 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.remote.provider.file; + +import io.airlift.configuration.Config; +import io.airlift.configuration.validation.FileExists; +import jakarta.validation.constraints.NotNull; + +import java.io.File; + +public class FileBasedRemoteS3ConnectionProviderConfig +{ + private File connectionsFile; + + @NotNull + @FileExists + public File getConnectionsFile() + { + return connectionsFile; + } + + @Config("remote-s3-connection-provider.connections-file-path") + public FileBasedRemoteS3ConnectionProviderConfig setConnectionsFile(File connectionsFile) + { + this.connectionsFile = connectionsFile; + return this; + } +} diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/file/README.md b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/file/README.md new file mode 100644 index 00000000..51367be0 --- /dev/null +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/file/README.md @@ -0,0 +1,53 @@ +# FileBasedRemoteS3ConnectionProvider Plugin + +## Overview + +The `FileBasedRemoteS3ConnectionProvider` plugin reads remote S3 connection details from a JSON file. This plugin is configured via a file path and supports a flexible JSON mapping +of access keys to connection details. + +## Configuration + +The following property is available for the `FileBasedRemoteS3ConnectionProvider`: + +| Property | Description | Default Value | +|-----------------------------------|-----------------------------------------------------------------|---------------| +| `remote-s3.connections-file-path` | The path to the JSON file containing the S3 connection mapping. | None | + +## Example Configuration + +Below is an example configuration for the `FileBasedRemoteS3ConnectionProvider`: + +```properties +remote-s3-connection-provider.type=file +remote-s3.connections-file-path=/path/to/your/connections.json +``` + +## JSON File Format + +The JSON file should map an emulated access key to its corresponding S3 connection details. For example: + +```json +{ + "emulated-access-key-1": { + "remoteCredential": { + "accessKey": "remote-access-key", + "secretKey": "remote-secret-key" + }, + "remoteSessionRole": { + "region": "us-east-1", + "roleArn": "arn:aws:iam::123456789012:role/role-name", + "externalId": "external-id", + "stsEndpoint": "https://sts.us-east-1.amazonaws.com" + }, + "remoteS3FacadeConfiguration": { + "remoteS3.https": true, + "remoteS3.domain": "s3.amazonaws.com", + "remoteS3.port": 443, + "remoteS3.virtual-host-style": false, + "remoteS3.hostname.template": "${domain}" + } + } +} +``` + +// ...existing content if any... diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/ForHttpRemoteS3ConnectionProvider.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/ForHttpRemoteS3ConnectionProvider.java new file mode 100644 index 00000000..24d97f00 --- /dev/null +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/ForHttpRemoteS3ConnectionProvider.java @@ -0,0 +1,31 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.remote.provider.http; + +import com.google.inject.BindingAnnotation; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@BindingAnnotation +@Target({FIELD, PARAMETER, METHOD}) +@Retention(RUNTIME) +public @interface ForHttpRemoteS3ConnectionProvider +{ +} diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/HttpRemoteS3ConnectionProvider.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/HttpRemoteS3ConnectionProvider.java new file mode 100644 index 00000000..61853565 --- /dev/null +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/HttpRemoteS3ConnectionProvider.java @@ -0,0 +1,124 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.remote.provider.http; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.benmanes.caffeine.cache.LoadingCache; +import com.google.common.annotations.VisibleForTesting; +import com.google.inject.Inject; +import io.airlift.http.client.FullJsonResponseHandler.JsonResponse; +import io.airlift.http.client.HttpClient; +import io.airlift.http.client.HttpStatus; +import io.airlift.json.JsonCodec; +import io.trino.aws.proxy.server.remote.provider.SerializableRemoteS3Connection; +import io.trino.aws.proxy.spi.credentials.Identity; +import io.trino.aws.proxy.spi.remote.RemoteS3Connection; +import io.trino.aws.proxy.spi.remote.RemoteS3ConnectionProvider; +import io.trino.aws.proxy.spi.rest.ParsedS3Request; +import io.trino.aws.proxy.spi.signing.SigningMetadata; +import jakarta.ws.rs.core.UriBuilder; + +import java.net.URI; +import java.net.URLEncoder; +import java.util.EnumSet; +import java.util.Locale; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.Set; + +import static com.github.benmanes.caffeine.cache.Caffeine.newBuilder; +import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static io.airlift.http.client.FullJsonResponseHandler.createFullJsonResponseHandler; +import static io.airlift.http.client.Request.Builder.prepareGet; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Objects.requireNonNull; + +public class HttpRemoteS3ConnectionProvider + implements RemoteS3ConnectionProvider +{ + private final HttpClient httpClient; + private final URI endpoint; + private final EnumSet requestQueryFields; + private final JsonCodec responseCodec; + private final ObjectMapper objectMapper; + private final Optional>, Optional>> cache; + + @Inject + public HttpRemoteS3ConnectionProvider( + @ForHttpRemoteS3ConnectionProvider HttpClient httpClient, + HttpRemoteS3ConnectionProviderConfig config, + JsonCodec responseCodec, + ObjectMapper objectMapper) + { + this.httpClient = requireNonNull(httpClient, "httpClient is null"); + this.responseCodec = requireNonNull(responseCodec, "responseCodec is null"); + this.endpoint = config.getEndpoint(); + this.requestQueryFields = config.getRequestFields(); + this.objectMapper = requireNonNull(objectMapper, "objectMapper is null"); + if (config.getCacheSize() > 0) { + this.cache = Optional.of(newBuilder() + .maximumSize(config.getCacheSize()) + .expireAfterWrite(config.getCacheTtl().toJavaTime()) + .build(this::requestRemoteConnection)); + } + else { + this.cache = Optional.empty(); + } + } + + @Override + public Optional remoteConnection(SigningMetadata signingMetadata, Optional identity, ParsedS3Request request) + { + Set> requestQueries = buildRequestQueries(signingMetadata, identity, request); + return cache.map(actualCache -> actualCache.get(requestQueries)) + .orElseGet(() -> requestRemoteConnection(requestQueries)); + } + + @VisibleForTesting + void resetCache() + { + cache.ifPresent(instantiatedCache -> { + instantiatedCache.invalidateAll(); + instantiatedCache.cleanUp(); + }); + } + + private Set> buildRequestQueries(SigningMetadata signingMetadata, Optional identity, ParsedS3Request request) + { + return requestQueryFields.stream() + .map(requestField -> Map.entry(requestField.toString(), requestField.getValue(signingMetadata, identity, request, objectMapper))) + .collect(toImmutableSet()); + } + + private Optional requestRemoteConnection(Set> requestQueryParams) + { + UriBuilder uriBuilder = UriBuilder.fromUri(endpoint); + requestQueryParams.forEach(param -> uriBuilder.queryParam(param.getKey().toLowerCase(Locale.ROOT), URLEncoder.encode(param.getValue(), UTF_8))); + JsonResponse response = httpClient.execute( + prepareGet().setUri(uriBuilder.build()).build(), + createFullJsonResponseHandler(responseCodec)); + HttpStatus statusCode = HttpStatus.fromStatusCode(response.getStatusCode()); + if (statusCode.family() != HttpStatus.Family.SUCCESSFUL) { + if (statusCode == HttpStatus.NOT_FOUND) { + return Optional.empty(); + } + throw new RuntimeException("Failed to get remote S3 connection with HTTP plugin. Response code: " + statusCode + "; body: \n" + response.getResponseBody()); + } + if (!response.hasValue()) { + throw new RuntimeException("Failed to get remote S3 connection with HTTP plugin. Response code: " + statusCode + "; no body", response.getException()); + } + return Optional.of(response.getValue()); + } +} diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/HttpRemoteS3ConnectionProviderConfig.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/HttpRemoteS3ConnectionProviderConfig.java new file mode 100644 index 00000000..5dc99914 --- /dev/null +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/HttpRemoteS3ConnectionProviderConfig.java @@ -0,0 +1,90 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.remote.provider.http; + +import io.airlift.configuration.Config; +import io.airlift.units.Duration; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; + +import java.net.URI; +import java.util.EnumSet; +import java.util.List; + +public class HttpRemoteS3ConnectionProviderConfig +{ + private URI endpoint; + private EnumSet requestFields = EnumSet.allOf(RequestQuery.class); + private long cacheSize; + private Duration cacheTtl = Duration.valueOf("1s"); + + @NotNull + public URI getEndpoint() + { + return endpoint; + } + + @Config("remote-s3-connection-provider.http.endpoint") + public HttpRemoteS3ConnectionProviderConfig setEndpoint(URI endpoint) + { + this.endpoint = endpoint; + return this; + } + + @NotNull + @NotEmpty + public EnumSet getRequestFields() + { + return requestFields; + } + + @Config("remote-s3-connection-provider.http.request-fields") + public HttpRemoteS3ConnectionProviderConfig setRequestFields(List requestFields) + { + if (requestFields.isEmpty()) { + this.requestFields = EnumSet.noneOf(RequestQuery.class); + } + else { + this.requestFields = EnumSet.copyOf(requestFields); + } + return this; + } + + @Min(0) + public long getCacheSize() + { + return cacheSize; + } + + @Config("remote-s3-connection-provider.http.cache-size") + public HttpRemoteS3ConnectionProviderConfig setCacheSize(long cacheSize) + { + this.cacheSize = cacheSize; + return this; + } + + @NotNull + public Duration getCacheTtl() + { + return cacheTtl; + } + + @Config("remote-s3-connection-provider.http.cache-ttl") + public HttpRemoteS3ConnectionProviderConfig setCacheTtl(Duration cacheTtl) + { + this.cacheTtl = cacheTtl; + return this; + } +} diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/HttpRemoteS3ConnectionProviderModule.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/HttpRemoteS3ConnectionProviderModule.java new file mode 100644 index 00000000..ee6c4e1e --- /dev/null +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/HttpRemoteS3ConnectionProviderModule.java @@ -0,0 +1,43 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.remote.provider.http; + +import com.google.inject.Binder; +import io.airlift.configuration.AbstractConfigurationAwareModule; +import io.trino.aws.proxy.server.remote.provider.SerializableRemoteS3Connection; + +import static io.airlift.configuration.ConfigBinder.configBinder; +import static io.airlift.http.client.HttpClientBinder.httpClientBinder; +import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; +import static io.trino.aws.proxy.spi.plugin.TrinoAwsProxyServerBinding.remoteS3ConnectionProviderModule; + +public class HttpRemoteS3ConnectionProviderModule + extends AbstractConfigurationAwareModule +{ + public static final String HTTP_REMOTE_S3_CONNECTION_PROVIDER = "http"; + + @Override + protected void setup(Binder binder) + { + install(remoteS3ConnectionProviderModule( + HTTP_REMOTE_S3_CONNECTION_PROVIDER, + HttpRemoteS3ConnectionProvider.class, + innerBinder -> { + httpClientBinder(innerBinder).bindHttpClient("remote-s3-connection-provider.http", ForHttpRemoteS3ConnectionProvider.class); + configBinder(innerBinder).bindConfig(HttpRemoteS3ConnectionProviderConfig.class); + innerBinder.bind(HttpRemoteS3ConnectionProvider.class); + jsonCodecBinder(innerBinder).bindJsonCodec(SerializableRemoteS3Connection.class); + })); + } +} diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/README.md b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/README.md new file mode 100644 index 00000000..9a4f1817 --- /dev/null +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/README.md @@ -0,0 +1,131 @@ +# HttpRemoteS3ConnectionProvider Plugin + +## Overview + +The `HttpRemoteS3ConnectionProvider` plugin provides a way to retrieve remote S3 connection details via HTTP requests. This plugin is configurable and supports caching of the +connection details. + +## Configuration + +The following table lists the configuration properties available for the `HttpRemoteS3ConnectionProvider`: + +| Property | Description | Default Value | +|-----------------------------------------------------|-----------------------------------------------------------------|---------------| +| `remote-s3-connection-provider.http.endpoint` | The HTTP endpoint to retrieve the remote S3 connection details. | None | +| `remote-s3-connection-provider.http.request-fields` | The fields to include in the HTTP request query parameters. | All fields | +| `remote-s3-connection-provider.http.cache-size` | The maximum size of the cache for remote S3 connections. | 0 | +| `remote-s3-connection-provider.http.cache-ttl` | The time-to-live for cache entries. | 1s | + +## Example Configuration + +Here is an example configuration for the `HttpRemoteS3ConnectionProvider`: + +```properties +remote-s3-connection-provider.type=http +remote-s3-connection-provider.http.endpoint=https://example.com/api/v1 +remote-s3-connection-provider.http.request-fields=BUCKET,EMULATED_ACCESS_KEY +remote-s3-connection-provider.http.cache-size=100 +remote-s3-connection-provider.http.cache-ttl=5m +``` + +## RequestQuery + +The `RequestQuery` enum defines the fields that can be included in the HTTP request query parameters. Each field is associated with a `FieldSelector` that determines how the value +for the field is obtained. The available fields are: + +- `BUCKET`: The bucket name from the `ParsedS3Request`. +- `KEY`: The key in the bucket from the `ParsedS3Request`. +- `EMULATED_ACCESS_KEY`: The access key from the `SigningMetadata`. +- `IDENTITY`: The identity in JSON format, if available. + +## OpenAPI Specification + +The following OpenAPI specification defines the API for retrieving remote S3 connection details: + +```yaml +openapi: 3.0.3 +info: + title: Remote S3 Connection Service + description: API for retrieving remote S3 connection details + version: 1.0.0 +servers: + - url: http://localhost:8080 +paths: + /: + get: + summary: Get Remote S3 Connection + description: Retrieve the remote S3 connection details based on query parameters + parameters: + - in: query + name: bucket + schema: + type: string + required: true + description: The bucket name + - in: query + name: key + schema: + type: string + required: true + description: The key in the bucket + - in: query + name: emulatedAccessKey + schema: + type: string + required: true + description: The emulated access key + - in: query + name: identity + schema: + type: string + required: false + description: The identity in JSON format + responses: + '200': + description: Successful response with remote S3 connection details + content: + application/json: + schema: + $ref: '#/components/schemas/RemoteS3Connection' + '404': + description: Remote S3 connection not found + '500': + description: Internal server error +components: + schemas: + RemoteS3Connection: + type: object + properties: + remoteCredential: + $ref: '#/components/schemas/Credential' + remoteSessionRole: + $ref: '#/components/schemas/RemoteSessionRole' + remoteS3FacadeConfiguration: + type: object + additionalProperties: + type: string + Credential: + type: object + properties: + accessKey: + type: string + secretKey: + type: string + session: + type: string + nullable: true + RemoteSessionRole: + type: object + properties: + region: + type: string + roleArn: + type: string + externalId: + type: string + nullable: true + stsEndpoint: + type: string + format: uri + nullable: true +``` diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/RequestQuery.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/RequestQuery.java new file mode 100644 index 00000000..c2595d2a --- /dev/null +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/RequestQuery.java @@ -0,0 +1,57 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.remote.provider.http; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.trino.aws.proxy.spi.credentials.Identity; +import io.trino.aws.proxy.spi.rest.ParsedS3Request; +import io.trino.aws.proxy.spi.signing.SigningMetadata; + +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +public enum RequestQuery +{ + BUCKET((_, _, request, _) -> request.bucketName()), + KEY((_, _, request, _) -> request.keyInBucket()), + EMULATED_ACCESS_KEY((signingMetadata, _, _, _) -> signingMetadata.credential().accessKey()), + IDENTITY((_, identity, _, objectMapper) -> identity.map(value -> { + try { + return objectMapper.writeValueAsString(value); + } + catch (JsonProcessingException exception) { + throw new RuntimeException(exception); + } + }).orElse("")); + + private final FieldSelector selector; + + RequestQuery(FieldSelector selector) + { + this.selector = requireNonNull(selector, "selector is null"); + } + + public String getValue(SigningMetadata signingMetadata, Optional identity, ParsedS3Request request, ObjectMapper objectMapper) + { + return selector.apply(signingMetadata, identity, request, objectMapper); + } + + @FunctionalInterface + private interface FieldSelector + { + String apply(SigningMetadata signingMetadata, Optional identity, ParsedS3Request request, ObjectMapper objectMapper); + } +} diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/preset/StaticRemoteS3ConnectionProviderConfig.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/preset/StaticRemoteS3ConnectionProviderConfig.java new file mode 100644 index 00000000..23229130 --- /dev/null +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/preset/StaticRemoteS3ConnectionProviderConfig.java @@ -0,0 +1,54 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.remote.provider.preset; + +import io.airlift.configuration.Config; +import io.airlift.configuration.ConfigSecuritySensitive; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; + +public class StaticRemoteS3ConnectionProviderConfig +{ + private String accessKey; + private String secretKey; + + @NotNull + @NotEmpty + public String getAccessKey() + { + return accessKey; + } + + @Config("remote-s3-connection-provider.access-key") + public StaticRemoteS3ConnectionProviderConfig setAccessKey(String accessKey) + { + this.accessKey = accessKey; + return this; + } + + @NotNull + @NotEmpty + public String getSecretKey() + { + return secretKey; + } + + @ConfigSecuritySensitive + @Config("remote-s3-connection-provider.secret-key") + public StaticRemoteS3ConnectionProviderConfig setSecretKey(String secretKey) + { + this.secretKey = secretKey; + return this; + } +} diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/preset/StaticRemoteS3ConnectionProviderModule.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/preset/StaticRemoteS3ConnectionProviderModule.java new file mode 100644 index 00000000..1a83db5f --- /dev/null +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/preset/StaticRemoteS3ConnectionProviderModule.java @@ -0,0 +1,46 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.remote.provider.preset; + +import com.google.inject.Binder; +import io.airlift.configuration.AbstractConfigurationAwareModule; +import io.trino.aws.proxy.spi.credentials.Credential; +import io.trino.aws.proxy.spi.plugin.config.RemoteS3ConnectionProviderConfig; +import io.trino.aws.proxy.spi.remote.RemoteS3Connection.StaticRemoteS3Connection; +import io.trino.aws.proxy.spi.remote.RemoteS3ConnectionProvider; + +import java.util.Optional; + +import static com.google.inject.multibindings.OptionalBinder.newOptionalBinder; +import static io.airlift.configuration.ConditionalModule.conditionalModule; + +public class StaticRemoteS3ConnectionProviderModule + extends AbstractConfigurationAwareModule +{ + public static final String STATIC_REMOTE_S3_CONNECTION_PROVIDER = "static"; + + @Override + protected void setup(Binder binder) + { + install(conditionalModule( + RemoteS3ConnectionProviderConfig.class, + config -> config.getPluginIdentifier().map(STATIC_REMOTE_S3_CONNECTION_PROVIDER::equals).orElse(false), + innerBinder -> { + StaticRemoteS3ConnectionProviderConfig config = buildConfigObject(StaticRemoteS3ConnectionProviderConfig.class); + newOptionalBinder(innerBinder, RemoteS3ConnectionProvider.class) + .setBinding() + .toInstance((_, _, _) -> Optional.of(new StaticRemoteS3Connection(new Credential(config.getAccessKey(), config.getSecretKey())))); + })); + } +} diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestProxiedEmulatedAndRemoteAssumedRoleRequests.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestProxiedEmulatedAndRemoteAssumedRoleRequests.java index c02b374b..14786c33 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestProxiedEmulatedAndRemoteAssumedRoleRequests.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestProxiedEmulatedAndRemoteAssumedRoleRequests.java @@ -17,7 +17,6 @@ import io.airlift.http.server.testing.TestingHttpServer; import io.trino.aws.proxy.server.testing.TestingCredentialsRolesProvider; import io.trino.aws.proxy.server.testing.TestingS3RequestRewriteController; -import io.trino.aws.proxy.server.testing.containers.S3Container; import io.trino.aws.proxy.server.testing.containers.S3Container.ForS3Container; import io.trino.aws.proxy.spi.credentials.Credential; import io.trino.aws.proxy.spi.credentials.IdentityCredential; @@ -29,6 +28,7 @@ import java.util.UUID; import static io.trino.aws.proxy.server.testing.TestingUtil.TESTING_IDENTITY_CREDENTIAL; +import static io.trino.aws.proxy.server.testing.containers.S3Container.POLICY_USER_CREDENTIAL; public class TestProxiedEmulatedAndRemoteAssumedRoleRequests extends TestProxiedAssumedRoleRequests @@ -41,15 +41,13 @@ public TestProxiedEmulatedAndRemoteAssumedRoleRequests( TestingCredentialsRolesProvider credentialsController, @ForS3Container S3Client storageClient, TrinoAwsProxyConfig trinoAwsProxyConfig, - S3Container s3Container, TestingS3RequestRewriteController requestRewriteController) { super(buildClient(httpServer, CREDENTIAL, trinoAwsProxyConfig.getS3Path(), trinoAwsProxyConfig.getStsPath()), credentialsController, storageClient, requestRewriteController); - Credential policyUserCredential = s3Container.policyUserCredential(); RemoteSessionRole remoteSessionRole = new RemoteSessionRole("us-east-1", "minio-doesnt-care", Optional.empty(), Optional.empty()); IdentityCredential identityCredential = new IdentityCredential(CREDENTIAL, TESTING_IDENTITY_CREDENTIAL.identity()); - credentialsController.addCredentials(identityCredential, new StaticRemoteS3Connection(policyUserCredential, remoteSessionRole)); + credentialsController.addCredentials(identityCredential, new StaticRemoteS3Connection(POLICY_USER_CREDENTIAL, remoteSessionRole)); } } diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestRemoteSessionProxiedRequests.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestRemoteSessionProxiedRequests.java index 683cfa67..0277a792 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestRemoteSessionProxiedRequests.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestRemoteSessionProxiedRequests.java @@ -17,7 +17,6 @@ import io.airlift.http.server.testing.TestingHttpServer; import io.trino.aws.proxy.server.testing.TestingCredentialsRolesProvider; import io.trino.aws.proxy.server.testing.TestingS3RequestRewriteController; -import io.trino.aws.proxy.server.testing.containers.S3Container; import io.trino.aws.proxy.server.testing.containers.S3Container.ForS3Container; import io.trino.aws.proxy.server.testing.harness.TrinoAwsProxyTest; import io.trino.aws.proxy.server.testing.harness.TrinoAwsProxyTestCommonModules.WithConfiguredBuckets; @@ -33,26 +32,26 @@ import static io.trino.aws.proxy.server.testing.TestingUtil.TESTING_IDENTITY_CREDENTIAL; import static io.trino.aws.proxy.server.testing.TestingUtil.clientBuilder; +import static io.trino.aws.proxy.server.testing.containers.S3Container.POLICY_USER_CREDENTIAL; @TrinoAwsProxyTest(filters = WithConfiguredBuckets.class) public class TestRemoteSessionProxiedRequests extends AbstractTestProxiedRequests { @Inject - public TestRemoteSessionProxiedRequests(@ForS3Container S3Client storageClient, S3Container s3Container, TestingCredentialsRolesProvider testingCredentialsRolesProvider, + public TestRemoteSessionProxiedRequests(@ForS3Container S3Client storageClient, TestingCredentialsRolesProvider testingCredentialsRolesProvider, TestingHttpServer httpServer, TrinoAwsProxyConfig trinoAwsProxyConfig, TestingS3RequestRewriteController requestRewriteController) { - super(buildClient(httpServer, trinoAwsProxyConfig, s3Container, testingCredentialsRolesProvider), storageClient, requestRewriteController); + super(buildClient(httpServer, trinoAwsProxyConfig, testingCredentialsRolesProvider), storageClient, requestRewriteController); } - private static S3Client buildClient(TestingHttpServer httpServer, TrinoAwsProxyConfig trinoAwsProxyConfig, S3Container s3Container, + private static S3Client buildClient(TestingHttpServer httpServer, TrinoAwsProxyConfig trinoAwsProxyConfig, TestingCredentialsRolesProvider testingCredentialsRolesProvider) { - Credential policyUserCredential = s3Container.policyUserCredential(); RemoteSessionRole remoteSessionRole = new RemoteSessionRole("us-east-1", "minio-doesnt-care", Optional.empty(), Optional.empty()); IdentityCredential identityCredential = new IdentityCredential(new Credential(UUID.randomUUID().toString(), UUID.randomUUID().toString()), TESTING_IDENTITY_CREDENTIAL.identity()); - testingCredentialsRolesProvider.addCredentials(identityCredential, new StaticRemoteS3Connection(policyUserCredential, remoteSessionRole)); + testingCredentialsRolesProvider.addCredentials(identityCredential, new StaticRemoteS3Connection(POLICY_USER_CREDENTIAL, remoteSessionRole)); AwsBasicCredentials awsBasicCredentials = AwsBasicCredentials.create(identityCredential.emulated().accessKey(), identityCredential.emulated().secretKey()); return clientBuilder(httpServer.getBaseUrl(), Optional.of(trinoAwsProxyConfig.getS3Path())) .credentialsProvider(() -> awsBasicCredentials) diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/provider/file/TestFileBasedRemoteS3ConnectionProvider.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/provider/file/TestFileBasedRemoteS3ConnectionProvider.java new file mode 100644 index 00000000..0cf9c44c --- /dev/null +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/provider/file/TestFileBasedRemoteS3ConnectionProvider.java @@ -0,0 +1,124 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.remote.provider.file; + +import com.google.inject.Inject; +import io.airlift.http.client.HttpStatus; +import io.airlift.http.server.testing.TestingHttpServer; +import io.trino.aws.proxy.server.AbstractTestProxiedRequests; +import io.trino.aws.proxy.server.TrinoAwsProxyConfig; +import io.trino.aws.proxy.server.remote.provider.file.TestFileBasedRemoteS3ConnectionProvider.Filter; +import io.trino.aws.proxy.server.testing.TestingCredentialsRolesProvider; +import io.trino.aws.proxy.server.testing.TestingS3RequestRewriteController; +import io.trino.aws.proxy.server.testing.TestingTrinoAwsProxyServer.Builder; +import io.trino.aws.proxy.server.testing.containers.S3Container.ForS3Container; +import io.trino.aws.proxy.server.testing.harness.BuilderFilter; +import io.trino.aws.proxy.server.testing.harness.TrinoAwsProxyTest; +import io.trino.aws.proxy.server.testing.harness.TrinoAwsProxyTestCommonModules.WithConfiguredBuckets; +import io.trino.aws.proxy.spi.credentials.Credential; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; +import jakarta.ws.rs.core.UriBuilder; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.S3Exception; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.util.UUID; + +import static io.trino.aws.proxy.server.testing.TestingUtil.TESTING_IDENTITY_CREDENTIAL; +import static io.trino.aws.proxy.server.testing.TestingUtil.clientBuilder; +import static io.trino.aws.proxy.server.testing.containers.S3Container.POLICY_USER_CREDENTIAL; +import static java.util.Objects.requireNonNull; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; + +@TrinoAwsProxyTest(filters = {WithConfiguredBuckets.class, Filter.class}) +public class TestFileBasedRemoteS3ConnectionProvider + extends AbstractTestProxiedRequests +{ + private static final Credential AWS_PROXY_CLIENT_CREDENTIAL = new Credential(UUID.randomUUID().toString(), UUID.randomUUID().toString()); + + public static final class Filter + implements BuilderFilter + { + @Override + public Builder filter(Builder builder) + { + File configFile; + try { + configFile = File.createTempFile("remote-s3-credentials", ".json"); + configFile.deleteOnExit(); + String jsonContent = """ + { + "%s": { + "remoteCredential": { + "accessKey": "%s", + "secretKey": "%s" + }, + "remoteSessionRole": { + "region": "us-east-1", + "roleArn": "minio-doesnt-care" + } + } + } + """.formatted(AWS_PROXY_CLIENT_CREDENTIAL.accessKey(), POLICY_USER_CREDENTIAL.accessKey(), POLICY_USER_CREDENTIAL.secretKey()); + Files.writeString(configFile.toPath(), jsonContent); + } + catch (IOException exception) { + throw new UncheckedIOException(exception); + } + + return builder + .withoutTestingRemoteS3ConnectionProvider() + .withProperty("remote-s3-connection-provider.type", "file") + .withProperty("remote-s3-connection-provider.connections-file-path", configFile.getAbsolutePath()); + } + } + + private final S3Client internalClient; + + @Inject + public TestFileBasedRemoteS3ConnectionProvider( + S3Client internalClient, + @ForS3Container S3Client storageClient, + TestingHttpServer httpServer, + TrinoAwsProxyConfig config, + TestingCredentialsRolesProvider testingCredentialsRolesProvider, + TestingS3RequestRewriteController requestRewriteController) + { + super(buildClient(httpServer, config), storageClient, requestRewriteController); + this.internalClient = requireNonNull(internalClient, "internalClient is null"); + testingCredentialsRolesProvider.addCredentials(new IdentityCredential(AWS_PROXY_CLIENT_CREDENTIAL, TESTING_IDENTITY_CREDENTIAL.identity())); + } + + private static S3Client buildClient(TestingHttpServer httpServer, TrinoAwsProxyConfig config) + { + AwsBasicCredentials awsBasicCredentials = AwsBasicCredentials.create(AWS_PROXY_CLIENT_CREDENTIAL.accessKey(), AWS_PROXY_CLIENT_CREDENTIAL.secretKey()); + return clientBuilder(UriBuilder.fromUri(httpServer.getBaseUrl()).path(config.getS3Path()).build()) + .credentialsProvider(() -> awsBasicCredentials) + .build(); + } + + @Test + public void testProxy404ResponseWhenRemoteS3ConnectionNotFound() + { + assertThatExceptionOfType(S3Exception.class) + .isThrownBy(internalClient::listBuckets) + .extracting(S3Exception::statusCode) + .isEqualTo(HttpStatus.NOT_FOUND.code()); + } +} diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/provider/file/TestFileBasedRemoteS3ConnectionProviderConfig.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/provider/file/TestFileBasedRemoteS3ConnectionProviderConfig.java new file mode 100644 index 00000000..7e478db6 --- /dev/null +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/provider/file/TestFileBasedRemoteS3ConnectionProviderConfig.java @@ -0,0 +1,45 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.remote.provider.file; + +import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.util.Map; + +import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; + +public class TestFileBasedRemoteS3ConnectionProviderConfig +{ + @Test + public void testExplicitPropertyMappings() + { + Map properties = ImmutableMap.builder() + .put("remote-s3-connection-provider.connections-file-path", "/dev/null") + .buildOrThrow(); + FileBasedRemoteS3ConnectionProviderConfig expected = new FileBasedRemoteS3ConnectionProviderConfig() + .setConnectionsFile(new File("/dev/null")); + assertFullMapping(properties, expected); + } + + @Test + public void testDefaults() + { + assertRecordedDefaults(recordDefaults(FileBasedRemoteS3ConnectionProviderConfig.class) + .setConnectionsFile(null)); + } +} diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/provider/http/TestHttpRemoteS3ConnectionProvider.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/provider/http/TestHttpRemoteS3ConnectionProvider.java new file mode 100644 index 00000000..c7c4a5a4 --- /dev/null +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/provider/http/TestHttpRemoteS3ConnectionProvider.java @@ -0,0 +1,150 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.remote.provider.http; + +import com.google.inject.Inject; +import io.airlift.http.client.HttpStatus; +import io.trino.aws.proxy.server.AbstractTestProxiedRequests; +import io.trino.aws.proxy.server.testing.TestingIdentity; +import io.trino.aws.proxy.server.testing.TestingS3RequestRewriteController; +import io.trino.aws.proxy.server.testing.containers.S3Container.ForS3Container; +import io.trino.aws.proxy.server.testing.harness.TrinoAwsProxyTest; +import io.trino.aws.proxy.server.testing.harness.TrinoAwsProxyTestCommonModules.WithConfiguredBuckets; +import io.trino.aws.proxy.spi.remote.RemoteS3ConnectionProvider; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.S3Exception; + +import static io.airlift.json.JsonCodec.jsonCodec; +import static io.trino.aws.proxy.server.testing.TestingUtil.TESTING_IDENTITY_CREDENTIAL; +import static io.trino.aws.proxy.server.testing.containers.S3Container.POLICY_USER_CREDENTIAL; +import static java.util.Objects.requireNonNull; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; + +@TrinoAwsProxyTest(filters = {WithConfiguredBuckets.class, WithHttpRemoteS3ConnectionProvider.class}) +public class TestHttpRemoteS3ConnectionProvider + extends AbstractTestProxiedRequests +{ + private final S3Client internalClient; + private final TestingHttpRemoteS3ConnectionProviderServlet testingHttpRemoteS3ConnectionProviderServlet; + private final HttpRemoteS3ConnectionProvider httpRemoteS3ConnectionProvider; + + @Inject + public TestHttpRemoteS3ConnectionProvider(S3Client internalClient, + @ForS3Container S3Client storageClient, + TestingS3RequestRewriteController requestRewriteController, + TestingHttpRemoteS3ConnectionProviderServlet testingHttpRemoteS3ConnectionProviderServlet, + RemoteS3ConnectionProvider httpRemoteS3ConnectionProvider) + { + super(internalClient, storageClient, requestRewriteController); + this.internalClient = requireNonNull(internalClient, "internalClient is null"); + this.testingHttpRemoteS3ConnectionProviderServlet = requireNonNull(testingHttpRemoteS3ConnectionProviderServlet, "testingHttpRemoteS3ConnectionProviderServlet is null"); + this.httpRemoteS3ConnectionProvider = (HttpRemoteS3ConnectionProvider) requireNonNull(httpRemoteS3ConnectionProvider, "httpRemoteS3ConnectionProvider is null"); + } + + @BeforeEach + public void cleanup() + { + httpRemoteS3ConnectionProvider.resetCache(); + testingHttpRemoteS3ConnectionProviderServlet.reset(); + testingHttpRemoteS3ConnectionProviderServlet.setResponse(""" + { + "remoteCredential": { + "accessKey": "%s", + "secretKey": "%s" + }, + "remoteSessionRole": { + "region": "us-east-1", + "roleArn": "minio-doesnt-care" + } + } + """.formatted(POLICY_USER_CREDENTIAL.accessKey(), POLICY_USER_CREDENTIAL.secretKey())); + } + + @AfterEach + public void checkRemoteS3ConnectionProviderHttpServletRequests() + { + assertThat(testingHttpRemoteS3ConnectionProviderServlet.getRequestParameters()).hasSizeGreaterThan(0).allSatisfy(queryParameters -> { + assertThat(queryParameters.get("emulated_access_key")).hasSize(1).first().isEqualTo(TESTING_IDENTITY_CREDENTIAL.emulated().accessKey()); + assertThat(queryParameters.get("identity")).hasSize(1).first() + .extracting(parameter -> jsonCodec(TestingIdentity.class).fromJson(parameter)) + .isEqualTo(TESTING_IDENTITY_CREDENTIAL.identity().orElseThrow()); + assertThat(queryParameters.get("key")).hasSize(1); + assertThat(queryParameters.get("bucket")).hasSize(1); + }); + } + + @Test + public void testProxy404ResponseWhenRemoteS3ConnectionNotFound() + { + testingHttpRemoteS3ConnectionProviderServlet.setResponseStatusOverride(HttpStatus.NOT_FOUND); + + assertThatExceptionOfType(S3Exception.class) + .isThrownBy(internalClient::listBuckets) + .extracting(S3Exception::statusCode) + .isEqualTo(HttpStatus.NOT_FOUND.code()); + } + + @Test + public void testProxy500ResponseWhenRemoteS3ConnectionProviderErrors() + { + testingHttpRemoteS3ConnectionProviderServlet.setResponseStatusOverride(HttpStatus.INTERNAL_SERVER_ERROR); + + assertThatExceptionOfType(S3Exception.class) + .isThrownBy(internalClient::listBuckets) + .extracting(S3Exception::statusCode) + .isEqualTo(HttpStatus.INTERNAL_SERVER_ERROR.code()); + } + + @Test + public void testProxy500ResponseWhenRemoteS3ConnectionProviderResponseMalformed() + { + testingHttpRemoteS3ConnectionProviderServlet.setResponse(""" + {"malformed": "response"} + """); + + assertThatExceptionOfType(S3Exception.class) + .isThrownBy(internalClient::listBuckets) + .extracting(S3Exception::statusCode) + .isEqualTo(HttpStatus.INTERNAL_SERVER_ERROR.code()); + } + + @Test + public void testProxy500ResponseWhenFacadeConfigsMalformed() + { + testingHttpRemoteS3ConnectionProviderServlet.setResponse(""" + { + "remoteCredential": { + "accessKey": "%s", + "secretKey": "%s" + }, + "remoteSessionRole": { + "region": "us-east-1", + "roleArn": "minio-doesnt-care" + }, + "remoteS3FacadeConfiguration": { + "some": "config" + } + } + """.formatted(POLICY_USER_CREDENTIAL.accessKey(), POLICY_USER_CREDENTIAL.secretKey())); + + assertThatExceptionOfType(S3Exception.class) + .isThrownBy(internalClient::listBuckets) + .extracting(S3Exception::statusCode) + .isEqualTo(HttpStatus.INTERNAL_SERVER_ERROR.code()); + } +} diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/provider/http/TestHttpRemoteS3ConnectionProviderConfig.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/provider/http/TestHttpRemoteS3ConnectionProviderConfig.java new file mode 100644 index 00000000..fa58e89c --- /dev/null +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/provider/http/TestHttpRemoteS3ConnectionProviderConfig.java @@ -0,0 +1,59 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.remote.provider.http; + +import com.google.common.collect.ImmutableMap; +import io.airlift.units.Duration; +import org.junit.jupiter.api.Test; + +import java.net.URI; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; + +public class TestHttpRemoteS3ConnectionProviderConfig +{ + @Test + public void testExplicitPropertyMappings() + throws Exception + { + Map properties = ImmutableMap.builder() + .put("remote-s3-connection-provider.http.endpoint", "http://localhost:8080") + .put("remote-s3-connection-provider.http.request-fields", "bucket,key,emulated-access-key") + .put("remote-s3-connection-provider.http.cache-size", "100") + .put("remote-s3-connection-provider.http.cache-ttl", "10s") + .buildOrThrow(); + HttpRemoteS3ConnectionProviderConfig expected = new HttpRemoteS3ConnectionProviderConfig() + .setEndpoint(new URI("http://localhost:8080")) + .setRequestFields(List.of(RequestQuery.BUCKET, RequestQuery.KEY, RequestQuery.EMULATED_ACCESS_KEY)) + .setCacheSize(100) + .setCacheTtl(Duration.valueOf("10s")); + assertFullMapping(properties, expected); + } + + @Test + public void testDefaults() + { + assertRecordedDefaults(recordDefaults(HttpRemoteS3ConnectionProviderConfig.class) + .setEndpoint(null) + .setRequestFields(EnumSet.allOf(RequestQuery.class).stream().collect(toImmutableList())) + .setCacheSize(0) + .setCacheTtl(Duration.valueOf("1s"))); + } +} diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/provider/http/TestHttpRemoteS3ConnectionProviderWithCustomFacade.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/provider/http/TestHttpRemoteS3ConnectionProviderWithCustomFacade.java new file mode 100644 index 00000000..97c8fec9 --- /dev/null +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/provider/http/TestHttpRemoteS3ConnectionProviderWithCustomFacade.java @@ -0,0 +1,179 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.remote.provider.http; + +import com.google.common.collect.ImmutableList; +import com.google.inject.BindingAnnotation; +import com.google.inject.Inject; +import io.trino.aws.proxy.server.AbstractTestProxiedRequests; +import io.trino.aws.proxy.server.remote.provider.http.TestHttpRemoteS3ConnectionProviderWithCustomFacade.WithSecondaryS3Container; +import io.trino.aws.proxy.server.testing.TestingS3RequestRewriteController; +import io.trino.aws.proxy.server.testing.TestingTrinoAwsProxyServer.Builder; +import io.trino.aws.proxy.server.testing.TestingUtil; +import io.trino.aws.proxy.server.testing.containers.S3Container; +import io.trino.aws.proxy.server.testing.containers.S3Container.ForS3Container; +import io.trino.aws.proxy.server.testing.harness.BuilderFilter; +import io.trino.aws.proxy.server.testing.harness.TrinoAwsProxyTest; +import io.trino.aws.proxy.server.testing.harness.TrinoAwsProxyTestCommonModules.WithConfiguredBuckets; +import io.trino.aws.proxy.spi.credentials.Credential; +import io.trino.aws.proxy.spi.remote.RemoteS3ConnectionProvider; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; + +import java.io.IOException; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; +import java.util.List; +import java.util.UUID; + +import static io.trino.aws.proxy.server.testing.TestingUtil.LOREM_IPSUM; +import static io.trino.aws.proxy.server.testing.TestingUtil.assertFileNotInS3; +import static io.trino.aws.proxy.server.testing.TestingUtil.getFileFromStorage; +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.RetentionPolicy.RUNTIME; +import static java.util.Objects.requireNonNull; +import static org.assertj.core.api.Assertions.assertThat; + +@TrinoAwsProxyTest(filters = {WithConfiguredBuckets.class, WithHttpRemoteS3ConnectionProvider.class, WithSecondaryS3Container.class}) +public class TestHttpRemoteS3ConnectionProviderWithCustomFacade + extends AbstractTestProxiedRequests +{ + public static final class WithSecondaryS3Container + implements BuilderFilter + { + @Override + public Builder filter(Builder builder) + { + Credential secondaryRemoteCredential = new Credential(UUID.randomUUID().toString(), UUID.randomUUID().toString()); + S3Container secondaryS3Container = new S3Container(ImmutableList.of(), secondaryRemoteCredential); + return builder.addModule(binder -> { + binder.bind(S3Container.class).annotatedWith(ForSecondaryS3Container.class).toInstance(secondaryS3Container); + binder.bind(S3Client.class).annotatedWith(ForSecondaryS3Container.class).toInstance(secondaryS3Container.get()); + binder.bind(Credential.class).annotatedWith(ForSecondaryS3Container.class).toInstance(secondaryRemoteCredential); + }); + } + } + + @Retention(RUNTIME) + @Target({FIELD, PARAMETER, METHOD}) + @BindingAnnotation + public @interface ForSecondaryS3Container + { + } + + private final S3Client internalClient; + private final S3Client remoteClient; + private final Credential originalRemoteCredential; + private final List initialBuckets; + private final S3Container secondaryS3Container; + private final S3Client secondaryRemoteClient; + private final Credential secondaryRemoteCredential; + private final TestingHttpRemoteS3ConnectionProviderServlet testingHttpRemoteS3ConnectionProviderServlet; + private final HttpRemoteS3ConnectionProvider httpRemoteS3ConnectionProvider; + + @Inject + public TestHttpRemoteS3ConnectionProviderWithCustomFacade(S3Client internalClient, + @ForS3Container S3Client remoteClient, + TestingS3RequestRewriteController requestRewriteController, + @ForS3Container Credential originalRemoteCredential, + @ForS3Container List initialBuckets, + @ForSecondaryS3Container S3Container secondaryS3Container, + @ForSecondaryS3Container S3Client secondaryRemoteClient, + @ForSecondaryS3Container Credential secondaryRemoteCredential, + TestingHttpRemoteS3ConnectionProviderServlet testingHttpRemoteS3ConnectionProviderServlet, + RemoteS3ConnectionProvider httpRemoteS3ConnectionProvider) + { + super(internalClient, secondaryRemoteClient, requestRewriteController); + + this.internalClient = requireNonNull(internalClient, "internalClient is null"); + this.remoteClient = requireNonNull(remoteClient, "remoteClient is null"); + this.originalRemoteCredential = requireNonNull(originalRemoteCredential, "originalCredential is null"); + this.initialBuckets = ImmutableList.copyOf(initialBuckets); + this.secondaryS3Container = requireNonNull(secondaryS3Container, "secondaryS3Container is null"); + this.secondaryRemoteClient = requireNonNull(secondaryRemoteClient, " secondaryRemoteClient is null"); + this.secondaryRemoteCredential = requireNonNull(secondaryRemoteCredential, "secondaryRemoteCredential is null"); + this.testingHttpRemoteS3ConnectionProviderServlet = requireNonNull(testingHttpRemoteS3ConnectionProviderServlet, "testingHttpRemoteS3ConnectionProviderServlet is null"); + this.httpRemoteS3ConnectionProvider = (HttpRemoteS3ConnectionProvider) requireNonNull(httpRemoteS3ConnectionProvider, "httpRemoteS3ConnectionProvider is null"); + + for (String bucket : this.initialBuckets) { + secondaryRemoteClient.createBucket(CreateBucketRequest.builder().bucket(bucket).build()); + } + } + + @BeforeEach + public void beforeEach() + { + testingHttpRemoteS3ConnectionProviderServlet.reset(); + testingHttpRemoteS3ConnectionProviderServlet.setResponse(""" + { + "remoteCredential": { + "accessKey": "%s", + "secretKey": "%s" + }, + "remoteS3FacadeConfiguration": { + "remoteS3.https": false, + "remoteS3.domain": "%s", + "remoteS3.port": "%s", + "remoteS3.virtual-host-style": false, + "remoteS3.hostname.template": "${domain}" + } + } + """.formatted(secondaryRemoteCredential.accessKey(), secondaryRemoteCredential.secretKey(), secondaryS3Container.containerHost().getHost(), + secondaryS3Container.containerHost().getPort())); + httpRemoteS3ConnectionProvider.resetCache(); + } + + @Test + public void testPutObjectInBothS3Containers() + throws IOException + { + String bucket = initialBuckets.getFirst(); + String objectKey = UUID.randomUUID().toString(); + + // When we upload an object to the proxy, it will first go to the secondary container (see bforeEaech setting the facade configuration) + internalClient.putObject(PutObjectRequest.builder().bucket(bucket).key(objectKey).build(), RequestBody.fromString(LOREM_IPSUM)); + + // Then files is in secondary S3 container + assertThat(getFileFromStorage(secondaryRemoteClient, bucket, objectKey)).isEqualTo(LOREM_IPSUM); + // Files not in first S3 container + assertFileNotInS3(remoteClient, bucket, objectKey); + + // Update remoteS3Connection to not override remoteS3FacadeConfig + testingHttpRemoteS3ConnectionProviderServlet.setResponse(""" + { + "remoteCredential": { + "accessKey": "%s", + "secretKey": "%s" + } + } + """.formatted(originalRemoteCredential.accessKey(), originalRemoteCredential.secretKey())); + httpRemoteS3ConnectionProvider.resetCache(); + + // File is not in aws-proxy + assertFileNotInS3(internalClient, bucket, objectKey); + // Put file again + internalClient.putObject(PutObjectRequest.builder().bucket(bucket).key(objectKey).build(), RequestBody.fromString(LOREM_IPSUM)); + // Files is in first remote S3 container + assertThat(getFileFromStorage(remoteClient, bucket, objectKey)).isEqualTo(LOREM_IPSUM); + + TestingUtil.cleanupBuckets(remoteClient); + TestingUtil.cleanupBuckets(secondaryRemoteClient); + } +} diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/provider/http/TestingHttpRemoteS3ConnectionProviderServlet.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/provider/http/TestingHttpRemoteS3ConnectionProviderServlet.java new file mode 100644 index 00000000..8aec9409 --- /dev/null +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/provider/http/TestingHttpRemoteS3ConnectionProviderServlet.java @@ -0,0 +1,83 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.remote.provider.http; + +import com.google.common.collect.Multimap; +import io.airlift.http.client.HttpStatus; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.google.common.collect.ImmutableListMultimap.flatteningToImmutableListMultimap; +import static java.util.Arrays.stream; + +public final class TestingHttpRemoteS3ConnectionProviderServlet + extends HttpServlet +{ + private final List> requestParameters = new ArrayList<>(); + + private Optional response = Optional.empty(); + private Optional responseStatus = Optional.empty(); + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) + throws IOException + { + requestParameters.add(req.getParameterMap().entrySet().stream().collect( + flatteningToImmutableListMultimap(Map.Entry::getKey, value -> stream(value.getValue())))); + + if (!req.getPathInfo().equals("/api/v1/remote_s3_connection")) { + resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found"); + return; + } + + if (responseStatus.isPresent()) { + HttpStatus actualResponseStatus = responseStatus.orElseThrow(); + resp.sendError(actualResponseStatus.code(), actualResponseStatus.reason()); + return; + } + + resp.setStatus(HttpServletResponse.SC_OK); + resp.setContentType("application/json"); + resp.getWriter().write(response.orElseThrow()); + } + + public void setResponse(String response) + { + this.response = Optional.of(response); + } + + public void setResponseStatusOverride(HttpStatus status) + { + this.responseStatus = Optional.of(status); + } + + public void reset() + { + requestParameters.clear(); + response = Optional.empty(); + responseStatus = Optional.empty(); + } + + public List> getRequestParameters() + { + return requestParameters; + } +} diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/provider/http/WithHttpRemoteS3ConnectionProvider.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/provider/http/WithHttpRemoteS3ConnectionProvider.java new file mode 100644 index 00000000..06d22db2 --- /dev/null +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/provider/http/WithHttpRemoteS3ConnectionProvider.java @@ -0,0 +1,50 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.remote.provider.http; + +import io.airlift.http.server.testing.TestingHttpServer; +import io.trino.aws.proxy.server.testing.TestingTrinoAwsProxyServer.Builder; +import io.trino.aws.proxy.server.testing.harness.BuilderFilter; +import jakarta.ws.rs.core.UriBuilder; + +import java.net.URI; + +import static io.trino.aws.proxy.server.testing.TestingUtil.createTestingHttpServer; + +public final class WithHttpRemoteS3ConnectionProvider + implements BuilderFilter +{ + @Override + public Builder filter(Builder builder) + { + TestingHttpRemoteS3ConnectionProviderServlet remoteS3ConnectionProviderServlet; + URI httpEndpointUri; + try { + remoteS3ConnectionProviderServlet = new TestingHttpRemoteS3ConnectionProviderServlet(); + TestingHttpServer server = createTestingHttpServer(remoteS3ConnectionProviderServlet); + server.start(); + httpEndpointUri = server.getBaseUrl(); + } + catch (Exception e) { + throw new RuntimeException("Failed to start test http remote s3 connection provider server", e); + } + + return builder.withoutTestingRemoteS3ConnectionProvider() + .withProperty("remote-s3-connection-provider.type", "http") + .withProperty("remote-s3-connection-provider.http.endpoint", UriBuilder.fromUri(httpEndpointUri).path("/api/v1/remote_s3_connection").build().toString()) + .withProperty("remote-s3-connection-provider.http.cache-size", "100") + .withProperty("remote-s3-connection-provider.http.cache-ttl", "5m") + .addModule(binder -> binder.bind(TestingHttpRemoteS3ConnectionProviderServlet.class).toInstance(remoteS3ConnectionProviderServlet)); + } +} diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/provider/preset/TestStaticRemoteS3ConnectionProvider.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/provider/preset/TestStaticRemoteS3ConnectionProvider.java new file mode 100644 index 00000000..5e3242a3 --- /dev/null +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/provider/preset/TestStaticRemoteS3ConnectionProvider.java @@ -0,0 +1,51 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.remote.provider.preset; + +import com.google.inject.Inject; +import io.trino.aws.proxy.server.AbstractTestProxiedRequests; +import io.trino.aws.proxy.server.testing.TestingS3RequestRewriteController; +import io.trino.aws.proxy.server.testing.TestingTrinoAwsProxyServer.Builder; +import io.trino.aws.proxy.server.testing.containers.S3Container.ForS3Container; +import io.trino.aws.proxy.server.testing.harness.BuilderFilter; +import io.trino.aws.proxy.server.testing.harness.TrinoAwsProxyTest; +import io.trino.aws.proxy.server.testing.harness.TrinoAwsProxyTestCommonModules.WithConfiguredBuckets; +import software.amazon.awssdk.services.s3.S3Client; + +import static io.trino.aws.proxy.server.testing.TestingUtil.TESTING_REMOTE_CREDENTIAL; + +@TrinoAwsProxyTest(filters = WithConfiguredBuckets.class) +public class TestStaticRemoteS3ConnectionProvider + extends AbstractTestProxiedRequests +{ + public static final class Filter + implements BuilderFilter + { + @Override + public Builder filter(Builder builder) + { + return builder + .withoutTestingRemoteS3ConnectionProvider() + .withProperty("remote-s3-connection-provider.type", "static") + .withProperty("remote-s3-connection-provider.access-key", TESTING_REMOTE_CREDENTIAL.accessKey()) + .withProperty("remote-s3-connection-provider.secret-key", TESTING_REMOTE_CREDENTIAL.secretKey()); + } + } + + @Inject + public TestStaticRemoteS3ConnectionProvider(S3Client s3Client, @ForS3Container S3Client storageClient, TestingS3RequestRewriteController requestRewriteController) + { + super(s3Client, storageClient, requestRewriteController); + } +} diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/provider/preset/TestStaticRemoteS3ConnectionProviderConfig.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/provider/preset/TestStaticRemoteS3ConnectionProviderConfig.java new file mode 100644 index 00000000..1b7efb16 --- /dev/null +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/provider/preset/TestStaticRemoteS3ConnectionProviderConfig.java @@ -0,0 +1,47 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.remote.provider.preset; + +import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; + +public class TestStaticRemoteS3ConnectionProviderConfig +{ + @Test + public void testExplicitPropertyMappings() + { + Map properties = ImmutableMap.builder() + .put("remote-s3-connection-provider.access-key", "access-key") + .put("remote-s3-connection-provider.secret-key", "secret-key") + .buildOrThrow(); + StaticRemoteS3ConnectionProviderConfig expected = new StaticRemoteS3ConnectionProviderConfig() + .setAccessKey("access-key") + .setSecretKey("secret-key"); + assertFullMapping(properties, expected); + } + + @Test + public void testConfigDefaults() + { + assertRecordedDefaults(recordDefaults(StaticRemoteS3ConnectionProviderConfig.class) + .setSecretKey(null) + .setAccessKey(null)); + } +} diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingCredentialsRolesProvider.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingCredentialsRolesProvider.java index 960ec7ad..57582380 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingCredentialsRolesProvider.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingCredentialsRolesProvider.java @@ -13,6 +13,7 @@ */ package io.trino.aws.proxy.server.testing; +import com.google.inject.Inject; import io.trino.aws.proxy.spi.credentials.AssumedRoleProvider; import io.trino.aws.proxy.spi.credentials.Credential; import io.trino.aws.proxy.spi.credentials.CredentialsProvider; @@ -20,6 +21,7 @@ import io.trino.aws.proxy.spi.credentials.Identity; import io.trino.aws.proxy.spi.credentials.IdentityCredential; import io.trino.aws.proxy.spi.remote.RemoteS3Connection; +import io.trino.aws.proxy.spi.remote.RemoteS3Connection.StaticRemoteS3Connection; import io.trino.aws.proxy.spi.remote.RemoteS3ConnectionProvider; import io.trino.aws.proxy.spi.rest.ParsedS3Request; import io.trino.aws.proxy.spi.signing.SigningMetadata; @@ -33,6 +35,8 @@ import java.util.concurrent.atomic.AtomicInteger; import static com.google.common.base.Preconditions.checkState; +import static io.trino.aws.proxy.server.testing.TestingUtil.TESTING_IDENTITY_CREDENTIAL; +import static io.trino.aws.proxy.server.testing.TestingUtil.TESTING_REMOTE_CREDENTIAL; import static java.util.Objects.requireNonNull; /** @@ -60,6 +64,13 @@ private record Session(Credential sessionCredential, String originalEmulatedAcce } } + @Inject + public TestingCredentialsRolesProvider() + { + addCredentials(TESTING_IDENTITY_CREDENTIAL); + setDefaultRemoteConnection(new StaticRemoteS3Connection(TESTING_REMOTE_CREDENTIAL)); + } + @Override public Optional remoteConnection(SigningMetadata signingMetadata, Optional identity, ParsedS3Request request) { diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingTrinoAwsProxyServer.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingTrinoAwsProxyServer.java index 7904f5b9..658a4bb1 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingTrinoAwsProxyServer.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingTrinoAwsProxyServer.java @@ -16,7 +16,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import com.google.inject.Inject; import com.google.inject.Injector; import com.google.inject.Key; import com.google.inject.Module; @@ -41,7 +40,6 @@ import io.trino.aws.proxy.server.testing.containers.S3Container.ForS3Container; import io.trino.aws.proxy.spi.credentials.Credential; import io.trino.aws.proxy.spi.credentials.IdentityCredential; -import io.trino.aws.proxy.spi.remote.RemoteS3Connection.StaticRemoteS3Connection; import java.io.Closeable; import java.util.Collection; @@ -98,6 +96,7 @@ public static class Builder private boolean v4PySparkContainerAdded; private boolean opaContainerAdded; private boolean addTestingCredentialsRoleProviders = true; + private boolean addTestingRemoteS3CredentialsProvider = true; public Builder addModule(Module module) { @@ -200,6 +199,12 @@ public Builder withoutTestingCredentialsRoleProviders() return this; } + public Builder withoutTestingRemoteS3ConnectionProvider() + { + addTestingRemoteS3CredentialsProvider = false; + return this; + } + public Builder withOpaContainer() { if (opaContainerAdded) { @@ -214,16 +219,15 @@ public Builder withOpaContainer() public TestingTrinoAwsProxyServer buildAndStart() { if (addTestingCredentialsRoleProviders) { - if (mockS3ContainerAdded) { - modules.add(binder -> binder.bind(TestingCredentialsInitializer.class).asEagerSingleton()); - } - addModule(credentialsProviderModule("testing", TestingCredentialsRolesProvider.class, (binder) -> binder.bind(TestingCredentialsRolesProvider.class).in(Scopes.SINGLETON))); withProperty("credentials-provider.type", "testing"); addModule(assumedRoleProviderModule("testing", TestingCredentialsRolesProvider.class, (binder) -> binder.bind(TestingCredentialsRolesProvider.class).in(Scopes.SINGLETON))); withProperty("assumed-role-provider.type", "testing"); + } + + if (addTestingRemoteS3CredentialsProvider) { addModule(remoteS3ConnectionProviderModule("testing", TestingCredentialsRolesProvider.class, - binder -> binder.bind(TestingCredentialsInitializer.class).in(Scopes.SINGLETON))); + binder -> binder.bind(TestingCredentialsRolesProvider.class).in(Scopes.SINGLETON))); withProperty("remote-s3-connection-provider.type", "testing"); } @@ -231,16 +235,6 @@ public TestingTrinoAwsProxyServer buildAndStart() } } - static class TestingCredentialsInitializer - { - @Inject - TestingCredentialsInitializer(TestingCredentialsRolesProvider credentialsController) - { - credentialsController.addCredentials(TESTING_IDENTITY_CREDENTIAL); - credentialsController.setDefaultRemoteConnection(new StaticRemoteS3Connection(TESTING_REMOTE_CREDENTIAL)); - } - } - private static TestingTrinoAwsProxyServer start(Collection extraModules, Map properties) { ImmutableList.Builder modules = ImmutableList.builder() diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/containers/S3Container.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/containers/S3Container.java index 27d0a6d4..7b5187f3 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/containers/S3Container.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/containers/S3Container.java @@ -49,6 +49,8 @@ public class S3Container public static final String POLICY_NAME = "managedPolicy"; + public static final Credential POLICY_USER_CREDENTIAL = new Credential(UUID.randomUUID().toString(), UUID.randomUUID().toString()); + private static final String IMAGE_NAME = "minio/minio"; private static final String IMAGE_TAG = "RELEASE.2024-07-15T19-02-30Z"; @@ -89,7 +91,6 @@ public class S3Container private final S3Client storageClient; private final List initialBuckets; private final Credential credential; - private final Credential policyUserCredential; @Override public S3Client get() @@ -130,8 +131,6 @@ public S3Container(@ForS3Container List initialBuckets, @ForS3Container .forcePathStyle(true) .credentialsProvider(() -> AwsBasicCredentials.create(credential.accessKey(), credential.secretKey())) .build(); - - policyUserCredential = new Credential(UUID.randomUUID().toString(), UUID.randomUUID().toString()); } public URI endpoint() @@ -144,20 +143,15 @@ public HostAndPort containerHost() return HostAndPort.fromParts(container.getHost(), container.getFirstMappedPort()); } - public Credential policyUserCredential() - { - return policyUserCredential; - } - @PostConstruct public void setUp() { initialBuckets.forEach(bucket -> storageClient.createBucket(request -> request.bucket(bucket))); // the Minio client does not have APIs for IAM or STS - execInContainer("Could not create user in container", "mc", "admin", "user", "add", "local", policyUserCredential.accessKey(), policyUserCredential.secretKey()); + execInContainer("Could not create user in container", "mc", "admin", "user", "add", "local", POLICY_USER_CREDENTIAL.accessKey(), POLICY_USER_CREDENTIAL.secretKey()); execInContainer("Could not create policy in container", "mc", "admin", "policy", "create", "local", POLICY_NAME, "/root/policy.json"); - execInContainer("Could not attach policy in container", "mc", "admin", "policy", "attach", "local", POLICY_NAME, "--user", policyUserCredential.accessKey()); + execInContainer("Could not attach policy in container", "mc", "admin", "policy", "attach", "local", POLICY_NAME, "--user", POLICY_USER_CREDENTIAL.accessKey()); } @PreDestroy