From f8b4b17a0921ef274b86c2ea163c74b993fd0ac1 Mon Sep 17 00:00:00 2001 From: rich7420 Date: Tue, 9 Jun 2026 21:51:43 +0800 Subject: [PATCH 1/2] HDDS-15515. Support object Content-Type end-to-end in S3 Gateway The S3 Gateway never persisted an object's Content-Type: it was only forwarded to HeaderPreprocessor for SigV4 signing, GET did not set it from the object and HEAD hard-coded binary/octet-stream, so CopyObject could not preserve/replace it. Store the request Content-Type in the OM key metadata (mirroring the existing ETag-in-metadata convention), return it on GET/HEAD (default binary/octet-stream), and propagate it on CopyObject (COPY keeps the source value, REPLACE uses the request value). A user-supplied x-amz-meta-content-type is remapped to avoid colliding with the stored Content-Type. Fixes s3-tests test_object_copy_{retaining,replacing}_metadata and copy_verify_contenttype. --- .../ozone/s3/endpoint/EndpointBase.java | 51 ++++++++++++++----- .../ozone/s3/endpoint/ObjectEndpoint.java | 35 ++++++++++++- .../ozone/s3/endpoint/TestObjectGet.java | 24 +++++++++ .../ozone/s3/endpoint/TestObjectPut.java | 37 ++++++++++++++ 4 files changed, 134 insertions(+), 13 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index 224a823e14d4..cb95f01b05e8 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -118,6 +118,12 @@ public abstract class EndpointBase { protected static final String ETAG_CUSTOM = "etag-custom"; + // Metadata key under which the object's Content-Type is stored in the OM key + // metadata (the standard Content-Type header name). + protected static final String CONTENT_TYPE = HttpHeaders.CONTENT_TYPE; + // Key used to preserve a user-supplied "x-amz-meta-content-type" so it does + // not collide with the object's Content-Type stored under CONTENT_TYPE. + protected static final String CONTENT_TYPE_CUSTOM = "content-type-custom"; private static final ThreadLocal MD5_PROVIDER; private static final ThreadLocal SHA_256_PROVIDER; @@ -319,22 +325,35 @@ protected Map getCustomMetadataFromHeaders( } } - // If the request contains a custom metadata header "x-amz-meta-ETag", - // replace the metadata key to "etag-custom" to prevent key metadata collision with - // the ETag calculated by hashing the object when storing the key in OM table. - // The custom ETag metadata header will be rebuilt during the headObject operation. - if (customMetadata.containsKey(HttpHeaders.ETAG) - || customMetadata.containsKey(HttpHeaders.ETAG.toLowerCase())) { - String customETag = customMetadata.get(HttpHeaders.ETAG) != null ? - customMetadata.get(HttpHeaders.ETAG) : customMetadata.get(HttpHeaders.ETAG.toLowerCase()); - customMetadata.remove(HttpHeaders.ETAG); - customMetadata.remove(HttpHeaders.ETAG.toLowerCase()); - customMetadata.put(ETAG_CUSTOM, customETag); - } + // A user-supplied x-amz-meta-{etag,content-type} would collide with the + // system ETag / Content-Type stored under the same key in the OM key + // metadata. Remap them to dedicated custom keys; they are rebuilt on read + // in addCustomMetadataHeaders(). + remapReservedMetadataKey(customMetadata, HttpHeaders.ETAG, ETAG_CUSTOM); + remapReservedMetadataKey(customMetadata, CONTENT_TYPE, CONTENT_TYPE_CUSTOM); return customMetadata; } + /** + * If {@code metadata} contains a user-supplied value under {@code headerName} + * (in either the canonical or lower-case form), move it to {@code customKey} + * so it does not collide with the system value stored under + * {@code headerName}. The original header is rebuilt on read in + * {@link #addCustomMetadataHeaders}. + */ + private static void remapReservedMetadataKey(Map metadata, + String headerName, String customKey) { + String lowerName = headerName.toLowerCase(); + if (metadata.containsKey(headerName) || metadata.containsKey(lowerName)) { + String value = metadata.get(headerName) != null + ? metadata.get(headerName) : metadata.get(lowerName); + metadata.remove(headerName); + metadata.remove(lowerName); + metadata.put(customKey, value); + } + } + protected void addCustomMetadataHeaders( Response.ResponseBuilder responseBuilder, OzoneKey key) { @@ -343,10 +362,18 @@ protected void addCustomMetadataHeaders( if (entry.getKey().equals(ETAG)) { continue; } + // The object's Content-Type is returned as the standard Content-Type + // response header, not as a user-metadata header. + if (entry.getKey().equals(CONTENT_TYPE)) { + continue; + } String metadataKey = entry.getKey(); if (metadataKey.equals(ETAG_CUSTOM)) { // Rebuild the ETag custom metadata header metadataKey = ETAG.toLowerCase(); + } else if (metadataKey.equals(CONTENT_TYPE_CUSTOM)) { + // Rebuild the user-supplied content-type custom metadata header + metadataKey = CONTENT_TYPE.toLowerCase(); } responseBuilder .header(CUSTOM_METADATA_HEADER_PREFIX + metadataKey, diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index acf358e3ce80..62ec5defd429 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -119,6 +119,8 @@ public class ObjectEndpoint extends ObjectOperationHandler { private static final String BUCKET = "bucket"; private static final String PATH = "path"; + // Default Content-Type for objects stored without one, matching S3. + private static final String DEFAULT_CONTENT_TYPE = "binary/octet-stream"; private static final Logger LOG = LoggerFactory.getLogger(ObjectEndpoint.class); @@ -267,6 +269,7 @@ Response handlePutRequest(ObjectRequestContext context, String keyPath, InputStr Map customMetadata = getCustomMetadataFromHeaders(getHeaders().getRequestHeaders()); + putContentType(customMetadata); Map tags = getTaggingFromHeaders(getHeaders()); long putLength; @@ -463,6 +466,13 @@ Response handleGetRequest(ObjectRequestContext context, String keyPath) for (Map.Entry entry : overrideQueryParameter.entrySet()) { String headerValue = getHeaders().getHeaderString(entry.getKey()); + // Default the response Content-Type to the object's stored Content-Type + // (or the standard default) so it is always returned; a + // response-content-type query parameter still overrides it below. + if (headerValue == null + && HttpHeaders.CONTENT_TYPE.equals(entry.getKey())) { + headerValue = contentTypeOf(keyDetails); + } String queryValue = queryParams.getFirst(entry.getValue()); if (queryValue != null) { headerValue = queryValue; @@ -497,6 +507,25 @@ static void addLastModifiedDate( RFC1123Util.FORMAT.format(lastModificationTime)); } + /** + * Store the request's Content-Type in the key metadata so it can be returned + * on GET/HEAD and propagated on copy. The original value is preserved by + * {@link HeaderPreprocessor} as {@code X-Ozone-Original-Content-Type}. + */ + private void putContentType(Map metadata) { + String contentType = + getHeaders().getHeaderString(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE); + if (contentType != null) { + metadata.put(CONTENT_TYPE, contentType); + } + } + + /** Returns the object's stored Content-Type, or the default if absent. */ + private static String contentTypeOf(OzoneKey key) { + String contentType = key.getMetadata().get(CONTENT_TYPE); + return contentType != null ? contentType : DEFAULT_CONTENT_TYPE; + } + static void addTagCountIfAny( ResponseBuilder responseBuilder, OzoneKey key) { // See x-amz-tagging-count in https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html @@ -572,7 +601,7 @@ public Response head( ResponseBuilder response = Response.ok().status(HttpStatus.SC_OK) .header(HttpHeaders.CONTENT_LENGTH, key.getDataSize()) - .header(HttpHeaders.CONTENT_TYPE, "binary/octet-stream") + .header(HttpHeaders.CONTENT_TYPE, contentTypeOf(key)) .header(STORAGE_CLASS_HEADER, s3StorageType.toString()); addEntityTagHeader(response, key); @@ -675,6 +704,7 @@ public Response initializeMultipartUpload( Map customMetadata = getCustomMetadataFromHeaders(getHeaders().getRequestHeaders()); + putContentType(customMetadata); Map tags = getTaggingFromHeaders(getHeaders()); @@ -1089,6 +1119,9 @@ private CopyObjectResponse copyObject(OzoneVolume volume, } else if (metadataCopyDirective.equals(CopyDirective.REPLACE.name())) { // Replace the metadata with the metadata form the request headers customMetadata = getCustomMetadataFromHeaders(getHeaders().getRequestHeaders()); + // With REPLACE, the Content-Type comes from the copy request (if any), + // not from the source object. + putContentType(customMetadata); } else { OS3Exception ex = newError(INVALID_ARGUMENT, metadataCopyDirective); ex.setErrorMessage("An error occurred (InvalidArgument) " + diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java index 8f310b74062d..0a4f11dbde1e 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java @@ -52,6 +52,7 @@ import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientStub; import org.apache.hadoop.ozone.client.OzoneClientTestUtils; +import org.apache.hadoop.ozone.s3.HeaderPreprocessor; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.util.RFC1123Util; import org.junit.jupiter.api.BeforeEach; @@ -298,6 +299,29 @@ public void getStatusCode() throws IOException, OS3Exception { assertNull(response.getHeaderString(TAG_COUNT_HEADER)); } + @Test + public void getAndHeadReturnStoredContentType() + throws IOException, OS3Exception { + // PUT an object whose Content-Type is preserved as ORIGINAL_CONTENT_TYPE. + when(headers.getHeaderString(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE)) + .thenReturn(CONTENT_TYPE1); + assertSucceeds(() -> put(rest, BUCKET_NAME, "typed-key", CONTENT)); + when(headers.getHeaderString(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE)) + .thenReturn(null); + + // GET and HEAD return the stored Content-Type when the request carries none. + assertEquals(CONTENT_TYPE1, + get(rest, BUCKET_NAME, "typed-key").getHeaderString("Content-Type")); + assertEquals(CONTENT_TYPE1, + rest.head(BUCKET_NAME, "typed-key").getHeaderString("Content-Type")); + + // An object stored without a Content-Type falls back to the default. + assertEquals("binary/octet-stream", + get(rest, BUCKET_NAME, KEY_NAME).getHeaderString("Content-Type")); + assertEquals("binary/octet-stream", + rest.head(BUCKET_NAME, KEY_NAME).getHeaderString("Content-Type")); + } + private void setDefaultHeader() { doReturn(CONTENT_TYPE1) .when(headers).getHeaderString("Content-Type"); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java index c9313207c01f..2af6cbcfdcc2 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java @@ -83,6 +83,7 @@ import org.apache.hadoop.ozone.client.OzoneKeyDetails; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.s3.HeaderPreprocessor; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; import org.apache.hadoop.ozone.s3.util.S3Consts; @@ -381,6 +382,42 @@ void testCopyObject() throws Exception { () -> put(objectEndpoint, "nonexistent", KEY_NAME, CONTENT)); } + @Test + void testContentTypeStoredAndCopied() throws Exception { + // PUT with an explicit Content-Type (preserved by HeaderPreprocessor). + when(headers.getHeaderString(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE)) + .thenReturn("audio/mpeg"); + + assertSucceeds(() -> putObject(CONTENT)); + OzoneKeyDetails source = bucket.getKey(KEY_NAME); + assertEquals("audio/mpeg", + source.getMetadata().get(HttpHeaders.CONTENT_TYPE)); + + // COPY directive: destination keeps the source Content-Type, ignoring the + // Content-Type on the copy request. + when(headers.getHeaderString(CUSTOM_METADATA_COPY_DIRECTIVE_HEADER)) + .thenReturn("COPY"); + when(headers.getHeaderString(COPY_SOURCE_HEADER)) + .thenReturn(BUCKET_NAME + "/" + urlEncode(KEY_NAME)); + when(headers.getHeaderString(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE)) + .thenReturn("text/plain"); + + assertSucceeds(() -> put(objectEndpoint, DEST_BUCKET_NAME, DEST_KEY, CONTENT)); + assertEquals("audio/mpeg", + destBucket.getKey(DEST_KEY).getMetadata().get(HttpHeaders.CONTENT_TYPE)); + + // REPLACE directive: destination takes the Content-Type from the request. + when(headers.getHeaderString(CUSTOM_METADATA_COPY_DIRECTIVE_HEADER)) + .thenReturn("REPLACE"); + when(headers.getRequestHeaders()).thenReturn(new MultivaluedHashMap<>()); + when(headers.getHeaderString(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE)) + .thenReturn("application/json"); + + assertSucceeds(() -> put(objectEndpoint, DEST_BUCKET_NAME, DEST_KEY, CONTENT)); + assertEquals("application/json", + destBucket.getKey(DEST_KEY).getMetadata().get(HttpHeaders.CONTENT_TYPE)); + } + @Test public void testCopyObjectMessageDigestResetDuringException() throws Exception { assertSucceeds(() -> putObject(CONTENT)); From 4755ebdb7596c06139c04a8ec59dc0228a346d31 Mon Sep 17 00:00:00 2001 From: rich7420 Date: Mon, 22 Jun 2026 21:18:21 +0800 Subject: [PATCH 2/2] HDDS-15515. Address review: GET ignores request Content-Type; consolidate reserved-key remap; add MPU/HEAD tests - GET no longer reflects the request Content-Type header; it is based on the stored object Content-Type with response-content-type still overriding. - Consolidate the ETag/Content-Type reserved-key remap+rebuild into a single RESERVED_METADATA_KEYS table and drop the duplicate CONTENT_TYPE constant. - Add tests: HEAD x-amz-meta-content-type round-trip and CreateMultipartUpload Content-Type storage. --- .../ozone/s3/endpoint/EndpointBase.java | 56 +++++++++---------- .../ozone/s3/endpoint/ObjectEndpoint.java | 28 +++++----- .../endpoint/TestMultipartUploadComplete.java | 18 ++++++ .../ozone/s3/endpoint/TestObjectGet.java | 33 ++++++++++- .../ozone/s3/endpoint/TestObjectHead.java | 35 ++++++++++++ 5 files changed, 122 insertions(+), 48 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index cb95f01b05e8..052dd34faa55 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -48,6 +48,7 @@ import static org.apache.hadoop.ozone.s3.util.S3Utils.validateSignatureHeader; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; @@ -118,13 +119,23 @@ public abstract class EndpointBase { protected static final String ETAG_CUSTOM = "etag-custom"; - // Metadata key under which the object's Content-Type is stored in the OM key - // metadata (the standard Content-Type header name). - protected static final String CONTENT_TYPE = HttpHeaders.CONTENT_TYPE; - // Key used to preserve a user-supplied "x-amz-meta-content-type" so it does - // not collide with the object's Content-Type stored under CONTENT_TYPE. protected static final String CONTENT_TYPE_CUSTOM = "content-type-custom"; + // System metadata key -> custom key. A user x-amz-meta-{etag,content-type} + // collides with the system ETag / Content-Type stored under the same key, so + // it is remapped on write and rebuilt on read; the system value is returned + // via its own ETag / Content-Type response header. + private static final Map RESERVED_METADATA_KEYS = + ImmutableMap.of( + ETAG, ETAG_CUSTOM, + HttpHeaders.CONTENT_TYPE, CONTENT_TYPE_CUSTOM); + + // Custom key -> lower-cased header, to rebuild remapped user metadata on read. + private static final Map REBUILT_RESERVED_KEYS = + RESERVED_METADATA_KEYS.entrySet().stream() + .collect(ImmutableMap.toImmutableMap( + Map.Entry::getValue, e -> e.getKey().toLowerCase())); + private static final ThreadLocal MD5_PROVIDER; private static final ThreadLocal SHA_256_PROVIDER; @@ -325,22 +336,16 @@ protected Map getCustomMetadataFromHeaders( } } - // A user-supplied x-amz-meta-{etag,content-type} would collide with the - // system ETag / Content-Type stored under the same key in the OM key - // metadata. Remap them to dedicated custom keys; they are rebuilt on read - // in addCustomMetadataHeaders(). - remapReservedMetadataKey(customMetadata, HttpHeaders.ETAG, ETAG_CUSTOM); - remapReservedMetadataKey(customMetadata, CONTENT_TYPE, CONTENT_TYPE_CUSTOM); + // Remap user values that collide with a reserved system key. + RESERVED_METADATA_KEYS.forEach((headerName, customKey) -> + remapReservedMetadataKey(customMetadata, headerName, customKey)); return customMetadata; } /** - * If {@code metadata} contains a user-supplied value under {@code headerName} - * (in either the canonical or lower-case form), move it to {@code customKey} - * so it does not collide with the system value stored under - * {@code headerName}. The original header is rebuilt on read in - * {@link #addCustomMetadataHeaders}. + * Move a user value under {@code headerName} (canonical or lower-case) to + * {@code customKey} so it does not collide with the system value. */ private static void remapReservedMetadataKey(Map metadata, String headerName, String customKey) { @@ -359,22 +364,13 @@ protected void addCustomMetadataHeaders( Map metadata = key.getMetadata(); for (Map.Entry entry : metadata.entrySet()) { - if (entry.getKey().equals(ETAG)) { - continue; - } - // The object's Content-Type is returned as the standard Content-Type - // response header, not as a user-metadata header. - if (entry.getKey().equals(CONTENT_TYPE)) { - continue; - } String metadataKey = entry.getKey(); - if (metadataKey.equals(ETAG_CUSTOM)) { - // Rebuild the ETag custom metadata header - metadataKey = ETAG.toLowerCase(); - } else if (metadataKey.equals(CONTENT_TYPE_CUSTOM)) { - // Rebuild the user-supplied content-type custom metadata header - metadataKey = CONTENT_TYPE.toLowerCase(); + // System ETag / Content-Type are returned via their own response header. + if (RESERVED_METADATA_KEYS.containsKey(metadataKey)) { + continue; } + // Rebuild a remapped user value (e.g. content-type-custom -> content-type). + metadataKey = REBUILT_RESERVED_KEYS.getOrDefault(metadataKey, metadataKey); responseBuilder .header(CUSTOM_METADATA_HEADER_PREFIX + metadataKey, entry.getValue()); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index 62ec5defd429..72f0c672bd39 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -133,7 +133,6 @@ public class ObjectEndpoint extends ObjectOperationHandler { public ObjectEndpoint() { overrideQueryParameter = ImmutableMap.builder() - .put(HttpHeaders.CONTENT_TYPE, "response-content-type") .put(HttpHeaders.CONTENT_LANGUAGE, "response-content-language") .put(HttpHeaders.EXPIRES, "response-expires") .put(HttpHeaders.CACHE_CONTROL, "response-cache-control") @@ -464,15 +463,16 @@ Response handleGetRequest(ObjectRequestContext context, String keyPath) MultivaluedMap queryParams = getContext().getUriInfo().getQueryParameters(); + // Content-Type comes from the stored object, not the request header; + // response-content-type still overrides it. + String contentType = queryParams.getFirst("response-content-type"); + if (contentType == null) { + contentType = contentTypeOf(keyDetails); + } + responseBuilder.header(HttpHeaders.CONTENT_TYPE, contentType); + for (Map.Entry entry : overrideQueryParameter.entrySet()) { String headerValue = getHeaders().getHeaderString(entry.getKey()); - // Default the response Content-Type to the object's stored Content-Type - // (or the standard default) so it is always returned; a - // response-content-type query parameter still overrides it below. - if (headerValue == null - && HttpHeaders.CONTENT_TYPE.equals(entry.getKey())) { - headerValue = contentTypeOf(keyDetails); - } String queryValue = queryParams.getFirst(entry.getValue()); if (queryValue != null) { headerValue = queryValue; @@ -508,21 +508,20 @@ static void addLastModifiedDate( } /** - * Store the request's Content-Type in the key metadata so it can be returned - * on GET/HEAD and propagated on copy. The original value is preserved by - * {@link HeaderPreprocessor} as {@code X-Ozone-Original-Content-Type}. + * Store the request's Content-Type (preserved by {@link HeaderPreprocessor} + * as {@code X-Ozone-Original-Content-Type}) in the key metadata. */ private void putContentType(Map metadata) { String contentType = getHeaders().getHeaderString(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE); if (contentType != null) { - metadata.put(CONTENT_TYPE, contentType); + metadata.put(HttpHeaders.CONTENT_TYPE, contentType); } } /** Returns the object's stored Content-Type, or the default if absent. */ private static String contentTypeOf(OzoneKey key) { - String contentType = key.getMetadata().get(CONTENT_TYPE); + String contentType = key.getMetadata().get(HttpHeaders.CONTENT_TYPE); return contentType != null ? contentType : DEFAULT_CONTENT_TYPE; } @@ -1119,8 +1118,7 @@ private CopyObjectResponse copyObject(OzoneVolume volume, } else if (metadataCopyDirective.equals(CopyDirective.REPLACE.name())) { // Replace the metadata with the metadata form the request headers customMetadata = getCustomMetadataFromHeaders(getHeaders().getRequestHeaders()); - // With REPLACE, the Content-Type comes from the copy request (if any), - // not from the source object. + // REPLACE: Content-Type comes from the request, not the source. putContentType(customMetadata); } else { OS3Exception ex = newError(INVALID_ARGUMENT, metadataCopyDirective); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java index 53500bbd2e54..66baf871b5a0 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java @@ -41,6 +41,7 @@ import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientStub; +import org.apache.hadoop.ozone.s3.HeaderPreprocessor; import org.apache.hadoop.ozone.s3.endpoint.CompleteMultipartUploadRequest.Part; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; @@ -122,6 +123,23 @@ public void testMultipartWithCustomMetadata() throws Exception { assertEquals("custom-value2", headResponse.getHeaderString(CUSTOM_METADATA_HEADER_PREFIX + "custom-key2")); } + @Test + public void testMultipartStoresContentType() throws Exception { + String key = UUID.randomUUID().toString(); + + // Content-Type set on CreateMultipartUpload is preserved as + // ORIGINAL_CONTENT_TYPE and must survive to the completed object. + when(headers.getHeaderString(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE)) + .thenReturn("video/webm"); + + String uploadID = initiateMultipartUpload(key); + Part part1 = uploadPart(rest, OzoneConsts.S3_BUCKET, key, 1, uploadID, "Multipart Upload 1"); + completeMultipartUpload(rest, OzoneConsts.S3_BUCKET, key, uploadID, singletonList(part1)); + + assertEquals("video/webm", rest.head(OzoneConsts.S3_BUCKET, key) + .getHeaderString(HttpHeaders.CONTENT_TYPE)); + } + @Test public void testMultipartInvalidPartOrderError() throws Exception { diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java index 0a4f11dbde1e..54d28c08b8b3 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java @@ -217,7 +217,8 @@ public void inheritRequestHeader() throws IOException, OS3Exception { Response response = get(rest, BUCKET_NAME, KEY_NAME); - assertEquals(CONTENT_TYPE1, + // Content-Type is not inherited from the request; key1 has none stored. + assertEquals("binary/octet-stream", response.getHeaderString("Content-Type")); assertEquals(CONTENT_LANGUAGE1, response.getHeaderString("Content-Language")); @@ -302,14 +303,14 @@ public void getStatusCode() throws IOException, OS3Exception { @Test public void getAndHeadReturnStoredContentType() throws IOException, OS3Exception { - // PUT an object whose Content-Type is preserved as ORIGINAL_CONTENT_TYPE. + // Store a Content-Type, then read with no request Content-Type. when(headers.getHeaderString(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE)) .thenReturn(CONTENT_TYPE1); assertSucceeds(() -> put(rest, BUCKET_NAME, "typed-key", CONTENT)); when(headers.getHeaderString(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE)) .thenReturn(null); - // GET and HEAD return the stored Content-Type when the request carries none. + // GET and HEAD return the stored Content-Type. assertEquals(CONTENT_TYPE1, get(rest, BUCKET_NAME, "typed-key").getHeaderString("Content-Type")); assertEquals(CONTENT_TYPE1, @@ -322,6 +323,32 @@ public void getAndHeadReturnStoredContentType() rest.head(BUCKET_NAME, KEY_NAME).getHeaderString("Content-Type")); } + @Test + public void getIgnoresRequestContentTypeUsesStored() + throws IOException, OS3Exception { + when(headers.getHeaderString(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE)) + .thenReturn(CONTENT_TYPE1); + assertSucceeds(() -> put(rest, BUCKET_NAME, "typed-key", CONTENT)); + when(headers.getHeaderString(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE)) + .thenReturn(null); + + // A request Content-Type is ignored; the stored value is returned. + doReturn(CONTENT_TYPE2).when(headers).getHeaderString("Content-Type"); + assertEquals(CONTENT_TYPE1, + get(rest, BUCKET_NAME, "typed-key").getHeaderString("Content-Type")); + + // No stored Content-Type: request still ignored, default returned. + assertEquals("binary/octet-stream", + get(rest, BUCKET_NAME, KEY_NAME).getHeaderString("Content-Type")); + + // response-content-type still overrides. + MultivaluedMap queryParameter = + rest.getContext().getUriInfo().getQueryParameters(); + queryParameter.putSingle("response-content-type", CONTENT_TYPE2); + assertEquals(CONTENT_TYPE2, + get(rest, BUCKET_NAME, "typed-key").getHeaderString("Content-Type")); + } + private void setDefaultHeader() { doReturn(CONTENT_TYPE1) .when(headers).getHeaderString("Content-Type"); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java index f6795ba8f566..158d856f7d4d 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java @@ -24,6 +24,7 @@ import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertSucceeds; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.put; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.PRECOND_FAILED; +import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX; import static org.apache.hadoop.ozone.s3.util.S3Consts.IF_MATCH_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.IF_NONE_MATCH_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.IF_UNMODIFIED_SINCE_HEADER; @@ -42,6 +43,8 @@ import java.time.ZoneId; import java.time.format.DateTimeFormatter; import javax.ws.rs.core.HttpHeaders; +import javax.ws.rs.core.MultivaluedHashMap; +import javax.ws.rs.core.MultivaluedMap; import javax.ws.rs.core.Response; import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -49,6 +52,7 @@ import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientStub; +import org.apache.hadoop.ozone.s3.HeaderPreprocessor; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.util.RFC1123Util; import org.apache.http.HttpStatus; @@ -231,6 +235,37 @@ public void testHeadObjectIncludesTagCount() assertEquals("2", response.getHeaderString(TAG_COUNT_HEADER)); } + @Test + public void testHeadSeparatesUserContentTypeMetadataFromObjectContentType() + throws Exception { + String keyName = "typed-with-user-meta"; + String objectContentType = "image/jpeg"; + String userContentType = "user/custom-type"; + + // PUT with both the object's Content-Type and a colliding user + // x-amz-meta-content-type. + when(headers.getHeaderString(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE)) + .thenReturn(objectContentType); + MultivaluedMap requestHeaders = new MultivaluedHashMap<>(); + requestHeaders.putSingle( + CUSTOM_METADATA_HEADER_PREFIX + "content-type", userContentType); + when(headers.getRequestHeaders()).thenReturn(requestHeaders); + assertSucceeds(() -> put(keyEndpoint, bucketName, keyName, "head-content")); + + // The user value is remapped, so the object's Content-Type is preserved. + assertEquals(objectContentType, + bucket.getKey(keyName).getMetadata().get(HttpHeaders.CONTENT_TYPE)); + + // HEAD returns the object Content-Type as the standard header and the user + // value as x-amz-meta-content-type. + Response response = keyEndpoint.head(bucketName, keyName); + assertEquals(HttpStatus.SC_OK, response.getStatus()); + assertEquals(objectContentType, + response.getHeaderString(HttpHeaders.CONTENT_TYPE)); + assertEquals(userContentType, + response.getHeaderString(CUSTOM_METADATA_HEADER_PREFIX + "content-type")); + } + private byte[] createKey(String keyPath) throws IOException { byte[] bytes = RandomStringUtils.secure().nextAlphanumeric(32).getBytes(UTF_8); try (OutputStream out = bucket.createKey(keyPath, bytes.length)) {