diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index 224a823e14d4..052dd34faa55 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -48,6 +48,7 @@ import static org.apache.hadoop.ozone.s3.util.S3Utils.validateSignatureHeader; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; @@ -118,6 +119,22 @@ public abstract class EndpointBase { protected static final String ETAG_CUSTOM = "etag-custom"; + protected static final String CONTENT_TYPE_CUSTOM = "content-type-custom"; + + // System metadata key -> custom key. A user x-amz-meta-{etag,content-type} + // collides with the system ETag / Content-Type stored under the same key, so + // it is remapped on write and rebuilt on read; the system value is returned + // via its own ETag / Content-Type response header. + private static final Map RESERVED_METADATA_KEYS = + ImmutableMap.of( + ETAG, ETAG_CUSTOM, + HttpHeaders.CONTENT_TYPE, CONTENT_TYPE_CUSTOM); + + // Custom key -> lower-cased header, to rebuild remapped user metadata on read. + private static final Map REBUILT_RESERVED_KEYS = + RESERVED_METADATA_KEYS.entrySet().stream() + .collect(ImmutableMap.toImmutableMap( + Map.Entry::getValue, e -> e.getKey().toLowerCase())); private static final ThreadLocal MD5_PROVIDER; private static final ThreadLocal SHA_256_PROVIDER; @@ -319,35 +336,41 @@ protected Map getCustomMetadataFromHeaders( } } - // If the request contains a custom metadata header "x-amz-meta-ETag", - // replace the metadata key to "etag-custom" to prevent key metadata collision with - // the ETag calculated by hashing the object when storing the key in OM table. - // The custom ETag metadata header will be rebuilt during the headObject operation. - if (customMetadata.containsKey(HttpHeaders.ETAG) - || customMetadata.containsKey(HttpHeaders.ETAG.toLowerCase())) { - String customETag = customMetadata.get(HttpHeaders.ETAG) != null ? - customMetadata.get(HttpHeaders.ETAG) : customMetadata.get(HttpHeaders.ETAG.toLowerCase()); - customMetadata.remove(HttpHeaders.ETAG); - customMetadata.remove(HttpHeaders.ETAG.toLowerCase()); - customMetadata.put(ETAG_CUSTOM, customETag); - } + // Remap user values that collide with a reserved system key. + RESERVED_METADATA_KEYS.forEach((headerName, customKey) -> + remapReservedMetadataKey(customMetadata, headerName, customKey)); return customMetadata; } + /** + * Move a user value under {@code headerName} (canonical or lower-case) to + * {@code customKey} so it does not collide with the system value. + */ + private static void remapReservedMetadataKey(Map metadata, + String headerName, String customKey) { + String lowerName = headerName.toLowerCase(); + if (metadata.containsKey(headerName) || metadata.containsKey(lowerName)) { + String value = metadata.get(headerName) != null + ? metadata.get(headerName) : metadata.get(lowerName); + metadata.remove(headerName); + metadata.remove(lowerName); + metadata.put(customKey, value); + } + } + protected void addCustomMetadataHeaders( Response.ResponseBuilder responseBuilder, OzoneKey key) { Map metadata = key.getMetadata(); for (Map.Entry entry : metadata.entrySet()) { - if (entry.getKey().equals(ETAG)) { - continue; - } String metadataKey = entry.getKey(); - if (metadataKey.equals(ETAG_CUSTOM)) { - // Rebuild the ETag custom metadata header - metadataKey = ETAG.toLowerCase(); + // System ETag / Content-Type are returned via their own response header. + if (RESERVED_METADATA_KEYS.containsKey(metadataKey)) { + continue; } + // Rebuild a remapped user value (e.g. content-type-custom -> content-type). + metadataKey = REBUILT_RESERVED_KEYS.getOrDefault(metadataKey, metadataKey); responseBuilder .header(CUSTOM_METADATA_HEADER_PREFIX + metadataKey, entry.getValue()); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index acf358e3ce80..72f0c672bd39 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -119,6 +119,8 @@ public class ObjectEndpoint extends ObjectOperationHandler { private static final String BUCKET = "bucket"; private static final String PATH = "path"; + // Default Content-Type for objects stored without one, matching S3. + private static final String DEFAULT_CONTENT_TYPE = "binary/octet-stream"; private static final Logger LOG = LoggerFactory.getLogger(ObjectEndpoint.class); @@ -131,7 +133,6 @@ public class ObjectEndpoint extends ObjectOperationHandler { public ObjectEndpoint() { overrideQueryParameter = ImmutableMap.builder() - .put(HttpHeaders.CONTENT_TYPE, "response-content-type") .put(HttpHeaders.CONTENT_LANGUAGE, "response-content-language") .put(HttpHeaders.EXPIRES, "response-expires") .put(HttpHeaders.CACHE_CONTROL, "response-cache-control") @@ -267,6 +268,7 @@ Response handlePutRequest(ObjectRequestContext context, String keyPath, InputStr Map customMetadata = getCustomMetadataFromHeaders(getHeaders().getRequestHeaders()); + putContentType(customMetadata); Map tags = getTaggingFromHeaders(getHeaders()); long putLength; @@ -461,6 +463,14 @@ Response handleGetRequest(ObjectRequestContext context, String keyPath) MultivaluedMap queryParams = getContext().getUriInfo().getQueryParameters(); + // Content-Type comes from the stored object, not the request header; + // response-content-type still overrides it. + String contentType = queryParams.getFirst("response-content-type"); + if (contentType == null) { + contentType = contentTypeOf(keyDetails); + } + responseBuilder.header(HttpHeaders.CONTENT_TYPE, contentType); + for (Map.Entry entry : overrideQueryParameter.entrySet()) { String headerValue = getHeaders().getHeaderString(entry.getKey()); String queryValue = queryParams.getFirst(entry.getValue()); @@ -497,6 +507,24 @@ static void addLastModifiedDate( RFC1123Util.FORMAT.format(lastModificationTime)); } + /** + * Store the request's Content-Type (preserved by {@link HeaderPreprocessor} + * as {@code X-Ozone-Original-Content-Type}) in the key metadata. + */ + private void putContentType(Map metadata) { + String contentType = + getHeaders().getHeaderString(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE); + if (contentType != null) { + metadata.put(HttpHeaders.CONTENT_TYPE, contentType); + } + } + + /** Returns the object's stored Content-Type, or the default if absent. */ + private static String contentTypeOf(OzoneKey key) { + String contentType = key.getMetadata().get(HttpHeaders.CONTENT_TYPE); + return contentType != null ? contentType : DEFAULT_CONTENT_TYPE; + } + static void addTagCountIfAny( ResponseBuilder responseBuilder, OzoneKey key) { // See x-amz-tagging-count in https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html @@ -572,7 +600,7 @@ public Response head( ResponseBuilder response = Response.ok().status(HttpStatus.SC_OK) .header(HttpHeaders.CONTENT_LENGTH, key.getDataSize()) - .header(HttpHeaders.CONTENT_TYPE, "binary/octet-stream") + .header(HttpHeaders.CONTENT_TYPE, contentTypeOf(key)) .header(STORAGE_CLASS_HEADER, s3StorageType.toString()); addEntityTagHeader(response, key); @@ -675,6 +703,7 @@ public Response initializeMultipartUpload( Map customMetadata = getCustomMetadataFromHeaders(getHeaders().getRequestHeaders()); + putContentType(customMetadata); Map tags = getTaggingFromHeaders(getHeaders()); @@ -1089,6 +1118,8 @@ private CopyObjectResponse copyObject(OzoneVolume volume, } else if (metadataCopyDirective.equals(CopyDirective.REPLACE.name())) { // Replace the metadata with the metadata form the request headers customMetadata = getCustomMetadataFromHeaders(getHeaders().getRequestHeaders()); + // REPLACE: Content-Type comes from the request, not the source. + putContentType(customMetadata); } else { OS3Exception ex = newError(INVALID_ARGUMENT, metadataCopyDirective); ex.setErrorMessage("An error occurred (InvalidArgument) " + diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java index 53500bbd2e54..66baf871b5a0 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java @@ -41,6 +41,7 @@ import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientStub; +import org.apache.hadoop.ozone.s3.HeaderPreprocessor; import org.apache.hadoop.ozone.s3.endpoint.CompleteMultipartUploadRequest.Part; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; @@ -122,6 +123,23 @@ public void testMultipartWithCustomMetadata() throws Exception { assertEquals("custom-value2", headResponse.getHeaderString(CUSTOM_METADATA_HEADER_PREFIX + "custom-key2")); } + @Test + public void testMultipartStoresContentType() throws Exception { + String key = UUID.randomUUID().toString(); + + // Content-Type set on CreateMultipartUpload is preserved as + // ORIGINAL_CONTENT_TYPE and must survive to the completed object. + when(headers.getHeaderString(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE)) + .thenReturn("video/webm"); + + String uploadID = initiateMultipartUpload(key); + Part part1 = uploadPart(rest, OzoneConsts.S3_BUCKET, key, 1, uploadID, "Multipart Upload 1"); + completeMultipartUpload(rest, OzoneConsts.S3_BUCKET, key, uploadID, singletonList(part1)); + + assertEquals("video/webm", rest.head(OzoneConsts.S3_BUCKET, key) + .getHeaderString(HttpHeaders.CONTENT_TYPE)); + } + @Test public void testMultipartInvalidPartOrderError() throws Exception { diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java index 8f310b74062d..54d28c08b8b3 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java @@ -52,6 +52,7 @@ import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientStub; import org.apache.hadoop.ozone.client.OzoneClientTestUtils; +import org.apache.hadoop.ozone.s3.HeaderPreprocessor; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.util.RFC1123Util; import org.junit.jupiter.api.BeforeEach; @@ -216,7 +217,8 @@ public void inheritRequestHeader() throws IOException, OS3Exception { Response response = get(rest, BUCKET_NAME, KEY_NAME); - assertEquals(CONTENT_TYPE1, + // Content-Type is not inherited from the request; key1 has none stored. + assertEquals("binary/octet-stream", response.getHeaderString("Content-Type")); assertEquals(CONTENT_LANGUAGE1, response.getHeaderString("Content-Language")); @@ -298,6 +300,55 @@ public void getStatusCode() throws IOException, OS3Exception { assertNull(response.getHeaderString(TAG_COUNT_HEADER)); } + @Test + public void getAndHeadReturnStoredContentType() + throws IOException, OS3Exception { + // Store a Content-Type, then read with no request Content-Type. + when(headers.getHeaderString(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE)) + .thenReturn(CONTENT_TYPE1); + assertSucceeds(() -> put(rest, BUCKET_NAME, "typed-key", CONTENT)); + when(headers.getHeaderString(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE)) + .thenReturn(null); + + // GET and HEAD return the stored Content-Type. + assertEquals(CONTENT_TYPE1, + get(rest, BUCKET_NAME, "typed-key").getHeaderString("Content-Type")); + assertEquals(CONTENT_TYPE1, + rest.head(BUCKET_NAME, "typed-key").getHeaderString("Content-Type")); + + // An object stored without a Content-Type falls back to the default. + assertEquals("binary/octet-stream", + get(rest, BUCKET_NAME, KEY_NAME).getHeaderString("Content-Type")); + assertEquals("binary/octet-stream", + rest.head(BUCKET_NAME, KEY_NAME).getHeaderString("Content-Type")); + } + + @Test + public void getIgnoresRequestContentTypeUsesStored() + throws IOException, OS3Exception { + when(headers.getHeaderString(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE)) + .thenReturn(CONTENT_TYPE1); + assertSucceeds(() -> put(rest, BUCKET_NAME, "typed-key", CONTENT)); + when(headers.getHeaderString(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE)) + .thenReturn(null); + + // A request Content-Type is ignored; the stored value is returned. + doReturn(CONTENT_TYPE2).when(headers).getHeaderString("Content-Type"); + assertEquals(CONTENT_TYPE1, + get(rest, BUCKET_NAME, "typed-key").getHeaderString("Content-Type")); + + // No stored Content-Type: request still ignored, default returned. + assertEquals("binary/octet-stream", + get(rest, BUCKET_NAME, KEY_NAME).getHeaderString("Content-Type")); + + // response-content-type still overrides. + MultivaluedMap queryParameter = + rest.getContext().getUriInfo().getQueryParameters(); + queryParameter.putSingle("response-content-type", CONTENT_TYPE2); + assertEquals(CONTENT_TYPE2, + get(rest, BUCKET_NAME, "typed-key").getHeaderString("Content-Type")); + } + private void setDefaultHeader() { doReturn(CONTENT_TYPE1) .when(headers).getHeaderString("Content-Type"); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java index f6795ba8f566..158d856f7d4d 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java @@ -24,6 +24,7 @@ import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertSucceeds; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.put; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.PRECOND_FAILED; +import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX; import static org.apache.hadoop.ozone.s3.util.S3Consts.IF_MATCH_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.IF_NONE_MATCH_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.IF_UNMODIFIED_SINCE_HEADER; @@ -42,6 +43,8 @@ import java.time.ZoneId; import java.time.format.DateTimeFormatter; import javax.ws.rs.core.HttpHeaders; +import javax.ws.rs.core.MultivaluedHashMap; +import javax.ws.rs.core.MultivaluedMap; import javax.ws.rs.core.Response; import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -49,6 +52,7 @@ import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientStub; +import org.apache.hadoop.ozone.s3.HeaderPreprocessor; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.util.RFC1123Util; import org.apache.http.HttpStatus; @@ -231,6 +235,37 @@ public void testHeadObjectIncludesTagCount() assertEquals("2", response.getHeaderString(TAG_COUNT_HEADER)); } + @Test + public void testHeadSeparatesUserContentTypeMetadataFromObjectContentType() + throws Exception { + String keyName = "typed-with-user-meta"; + String objectContentType = "image/jpeg"; + String userContentType = "user/custom-type"; + + // PUT with both the object's Content-Type and a colliding user + // x-amz-meta-content-type. + when(headers.getHeaderString(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE)) + .thenReturn(objectContentType); + MultivaluedMap requestHeaders = new MultivaluedHashMap<>(); + requestHeaders.putSingle( + CUSTOM_METADATA_HEADER_PREFIX + "content-type", userContentType); + when(headers.getRequestHeaders()).thenReturn(requestHeaders); + assertSucceeds(() -> put(keyEndpoint, bucketName, keyName, "head-content")); + + // The user value is remapped, so the object's Content-Type is preserved. + assertEquals(objectContentType, + bucket.getKey(keyName).getMetadata().get(HttpHeaders.CONTENT_TYPE)); + + // HEAD returns the object Content-Type as the standard header and the user + // value as x-amz-meta-content-type. + Response response = keyEndpoint.head(bucketName, keyName); + assertEquals(HttpStatus.SC_OK, response.getStatus()); + assertEquals(objectContentType, + response.getHeaderString(HttpHeaders.CONTENT_TYPE)); + assertEquals(userContentType, + response.getHeaderString(CUSTOM_METADATA_HEADER_PREFIX + "content-type")); + } + private byte[] createKey(String keyPath) throws IOException { byte[] bytes = RandomStringUtils.secure().nextAlphanumeric(32).getBytes(UTF_8); try (OutputStream out = bucket.createKey(keyPath, bytes.length)) { diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java index c9313207c01f..2af6cbcfdcc2 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java @@ -83,6 +83,7 @@ import org.apache.hadoop.ozone.client.OzoneKeyDetails; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.s3.HeaderPreprocessor; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; import org.apache.hadoop.ozone.s3.util.S3Consts; @@ -381,6 +382,42 @@ void testCopyObject() throws Exception { () -> put(objectEndpoint, "nonexistent", KEY_NAME, CONTENT)); } + @Test + void testContentTypeStoredAndCopied() throws Exception { + // PUT with an explicit Content-Type (preserved by HeaderPreprocessor). + when(headers.getHeaderString(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE)) + .thenReturn("audio/mpeg"); + + assertSucceeds(() -> putObject(CONTENT)); + OzoneKeyDetails source = bucket.getKey(KEY_NAME); + assertEquals("audio/mpeg", + source.getMetadata().get(HttpHeaders.CONTENT_TYPE)); + + // COPY directive: destination keeps the source Content-Type, ignoring the + // Content-Type on the copy request. + when(headers.getHeaderString(CUSTOM_METADATA_COPY_DIRECTIVE_HEADER)) + .thenReturn("COPY"); + when(headers.getHeaderString(COPY_SOURCE_HEADER)) + .thenReturn(BUCKET_NAME + "/" + urlEncode(KEY_NAME)); + when(headers.getHeaderString(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE)) + .thenReturn("text/plain"); + + assertSucceeds(() -> put(objectEndpoint, DEST_BUCKET_NAME, DEST_KEY, CONTENT)); + assertEquals("audio/mpeg", + destBucket.getKey(DEST_KEY).getMetadata().get(HttpHeaders.CONTENT_TYPE)); + + // REPLACE directive: destination takes the Content-Type from the request. + when(headers.getHeaderString(CUSTOM_METADATA_COPY_DIRECTIVE_HEADER)) + .thenReturn("REPLACE"); + when(headers.getRequestHeaders()).thenReturn(new MultivaluedHashMap<>()); + when(headers.getHeaderString(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE)) + .thenReturn("application/json"); + + assertSucceeds(() -> put(objectEndpoint, DEST_BUCKET_NAME, DEST_KEY, CONTENT)); + assertEquals("application/json", + destBucket.getKey(DEST_KEY).getMetadata().get(HttpHeaders.CONTENT_TYPE)); + } + @Test public void testCopyObjectMessageDigestResetDuringException() throws Exception { assertSucceeds(() -> putObject(CONTENT));