Skip to content

HDDS-15740. Validate partNumber on GetObject and return InvalidPart for out-of-range parts #10670

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rich7420 wants to merge 1 commit into
base: master
Choose a base branch
from
+56 −20
Open

HDDS-15740. Validate partNumber on GetObject and return InvalidPart for out-of-range parts #10670

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 4 additions & 6 deletions hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -283,9 +283,8 @@ Create&Download big file by multipart upload and get file via part numbers

Should Be Equal As Integers 10000000 ${${part_1_size} + ${part_2_size}}

${get_part_3_response} Execute AWSS3APICli get-object --bucket ${BUCKET} --key big_file /tmp/big_file_3 --part-number 3
Should contain ${get_part_3_response} \"ContentLength\": 0
Should contain ${get_part_3_response} \"PartsCount\": 2
${get_part_3_response} Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key big_file /tmp/big_file_3 --part-number 3 255
Should contain ${get_part_3_response} InvalidPart
# clean up
Execute AWSS3Cli rm s3://${BUCKET}/big_file
Execute rm -rf /tmp/big_file
Expand All @@ -296,9 +295,8 @@ Create&Download big file by multipart upload and get file via part numbers
Create&Download big file by multipart upload and get file not existed part number
Execute head -c 10000000 </dev/urandom > /tmp/big_file
${result} Execute AWSS3CliDebug cp /tmp/big_file s3://${BUCKET}/
${get_part_99_response} Execute AWSS3APICli get-object --bucket ${BUCKET} --key big_file /tmp/big_file_1 --part-number 99
Should contain ${get_part_99_response} \"ContentLength\": 0
Should contain ${get_part_99_response} \"PartsCount\": 2
${get_part_99_response} Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key big_file /tmp/big_file_1 --part-number 99 255
Should contain ${get_part_99_response} InvalidPart
# clean up
Execute AWSS3Cli rm s3://${BUCKET}/big_file
Execute rm -rf /tmp/big_file
Expand Down
9 changes: 6 additions & 3 deletions ...tion-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1827,9 +1827,12 @@ public void testGetNotExistedPart(@TempDir Path tempDir) throws Exception {

GetObjectRequest getObjectRequestOne = new GetObjectRequest(bucketName, keyName);
getObjectRequestOne.setPartNumber(4);
S3Object s3ObjectOne = s3Client.getObject(getObjectRequestOne);
long partOneContentLength = s3ObjectOne.getObjectMetadata().getContentLength();
assertEquals(0, partOneContentLength);
// Reading a part number beyond the object's part count must fail with
// InvalidPart, instead of returning an empty (0-byte) object.
AmazonServiceException ase = assertThrows(AmazonServiceException.class,
() -> s3Client.getObject(getObjectRequestOne));
assertEquals(400, ase.getStatusCode());
assertEquals("InvalidPart", ase.getErrorCode());
}

@Test
Expand Down
30 changes: 26 additions & 4 deletions ...c/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1016,10 +1016,32 @@ void testGetNotExistedPart() throws Exception {

s3Bucket.completeMultipartUpload(keyName, uploadID, partsMap);

OzoneKeyDetails s3KeyDetailsWithNotExistedParts = ozClient.getProxy()
.getS3KeyDetails(s3Bucket.getName(), keyName, 4);
List<OzoneKeyLocation> ozoneKeyLocations = s3KeyDetailsWithNotExistedParts.getOzoneKeyLocations();
assertEquals(0, ozoneKeyLocations.size());
// Reading a part number beyond the object's part count must fail with
// InvalidPart, instead of silently returning an empty (0-byte) result.
OzoneTestUtils.expectOmException(OMException.ResultCodes.INVALID_PART, () ->
ozClient.getProxy().getS3KeyDetails(s3Bucket.getName(), keyName, 4));
}

@Test
void testGetPartNumberOnNonMultipartKey() throws Exception {
keyName = "non-multipart-file";
OzoneVolume s3volume = store.getVolume("s3v");
s3volume.createBucket(bucketName);
OzoneBucket s3Bucket = s3volume.getBucket(bucketName);

byte[] data = generateData(1024, (byte) 97);
try (OzoneOutputStream out = s3Bucket.createKey(keyName, data.length)) {
out.write(data);
}

// partNumber == 1 on a non-multipart key returns the whole object.
OzoneKeyDetails part1 =
ozClient.getProxy().getS3KeyDetails(bucketName, keyName, 1);
assertEquals(data.length, part1.getDataSize());

// partNumber > 1 on a non-multipart key is out of range.
OzoneTestUtils.expectOmException(OMException.ResultCodes.INVALID_PART, () ->
ozClient.getProxy().getS3KeyDetails(bucketName, keyName, 2));
}

private String verifyUploadedPart(String uploadID, String partName,
Expand Down
12 changes: 5 additions & 7 deletions ...p-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1523,13 +1523,11 @@ void testGetNotExistedPart() throws IOException {
.setKeyName(keyName)
.setMultipartUploadPartNumber(99)
.build();
OmKeyInfo omKeyInfo = keyManager.getKeyInfo(keyArgs, RESOLVED_BUCKET, "test");
assertEquals(keyName, omKeyInfo.getKeyName());
assertNotNull(omKeyInfo.getLatestVersionLocations());

List<OmKeyLocationInfo> locationList = omKeyInfo.getLatestVersionLocations().getLocationList();
assertNotNull(locationList);
assertEquals(0, locationList.size());
// Reading a part number beyond the object's part count must fail with
// InvalidPart, instead of returning an empty (0-byte) result.
OMException ex = assertThrows(OMException.class,
() -> keyManager.getKeyInfo(keyArgs, RESOLVED_BUCKET, "test"));
assertEquals(OMException.ResultCodes.INVALID_PART, ex.getResult());
}

private OmKeyInfo getMockedOmKeyInfo(OmBucketInfo bucketInfo, long parentId, String key, long objectId) {
Expand Down
15 changes: 15 additions & 0 deletions hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FILE_NOT_FOUND;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INTERNAL_ERROR;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_KMS_PROVIDER;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_PART;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.SCM_GET_PIPELINE_EXCEPTION;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND;
Expand Down Expand Up @@ -640,12 +641,26 @@ private OmKeyInfo readKeyInfo(OmKeyArgs args, BucketLayout bucketLayout)
.filter(it -> it.getPartNumber() == partNumberParam)
.collect(Collectors.toList());

// A requested part number that has no blocks does not exist in this
// object (part numbers may be non-contiguous), so it is out of range.
if (currentLocations.isEmpty()) {
throw new OMException("Cannot read part " + partNumberParam
+ " of key " + keyName + " because it does not exist",
INVALID_PART);
}

value.updateLocationInfoList(currentLocations, true, true);

value.setDataSize(currentLocations.stream()
.mapToLong(BlockLocationInfo::getLength)
.sum());
} else if (partNumberParam > 1) {
// Non-multipart key: only part number 1 (the whole object) is valid;
// any higher part number is out of range.
throw new OMException("Cannot read part " + partNumberParam
+ " of non-multipart key " + keyName, INVALID_PART);
}
// Non-multipart key with partNumber == 1 returns the whole object.
}
return value;
}
Expand Down