diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java index de8487de3293..81aea095d11b 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/OMMetadataManagerTestUtils.java @@ -268,6 +268,42 @@ public static void writeKeyToOm(OMMetadataManager omMetadataManager, .build()); } + /** + * Write a multi-block key on OM instance with an explicit replication config. + * Combines the location-group and replication-config overloads so tests can + * exercise a real (> 1) replication factor. + * @throw IOException while writing. + */ + @SuppressWarnings("checkstyle:parameternumber") + public static void writeKeyToOm(OMMetadataManager omMetadataManager, + String keyName, + String bucketName, + String volName, + String fileName, + long objectId, + long parentObjectId, + long bucketObjectId, + long volumeObjectId, + List locationVersions, + BucketLayout bucketLayout, + long dataSize, + ReplicationConfig replicationConfig) + throws IOException { + String omKey = getKey(omMetadataManager, keyName, bucketName, volName, + fileName, parentObjectId, bucketObjectId, volumeObjectId, bucketLayout); + omMetadataManager.getKeyTable(bucketLayout).put(omKey, + new OmKeyInfo.Builder() + .setBucketName(bucketName) + .setVolumeName(volName) + .setKeyName(keyName) + .setDataSize(dataSize) + .setOmKeyLocationInfos(locationVersions) + .setReplicationConfig(replicationConfig) + .setObjectID(objectId) + .setParentObjectID(parentObjectId) + .build()); + } + @SuppressWarnings("checkstyle:ParameterNumber") private static String getKey(OMMetadataManager omMetadataManager, String key, String bucket, String volume, String fileName, long parentObjectId, long bucketObjectId, long volumeObjectId, diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/AbstractTreeNSSummaryScenario.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/AbstractTreeNSSummaryScenario.java new file mode 100644 index 000000000000..114647c2c547 --- /dev/null +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/AbstractTreeNSSummaryScenario.java @@ -0,0 +1,806 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.api; + +import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE; +import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS; +import static org.apache.hadoop.ozone.om.helpers.QuotaUtil.getReplicatedSize; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm; +import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType.USER; +import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.WRITE; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.StorageType; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.ozone.OzoneAcl; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; +import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.recon.api.types.BucketObjectDBInfo; +import org.apache.hadoop.ozone.recon.api.types.DUResponse; +import org.apache.hadoop.ozone.recon.api.types.EntityType; +import org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse; +import org.apache.hadoop.ozone.recon.api.types.QuotaUsageResponse; +import org.apache.hadoop.ozone.recon.api.types.ResponseStatus; +import org.apache.hadoop.ozone.recon.api.types.VolumeObjectDBInfo; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; + +/** + * Shared tree fixture for the FSO and Legacy (filesystem-paths enabled) + * scenarios. The namespace is: + * 
+ *                vol
+ *             /       \
+ *        bucket1      bucket2
+ *        /    \         /    \
+ *     file1    dir1    file4  file5
+ *           /   \   \   \
+ *        dir2  dir3 dir4 file7
+ *         /     \      \
+ *       file2   file3  file6
+ *  ----------------------------------------
+ *                  vol2
+ *              /         \
+ *      bucket3          bucket4
+ *      /      \           /
+ *   file8     dir5      file11
+ *            /    \
+ *        file9    file10
+ *  ----------------------------------------
+ *                    vol3
+ *                     |
+ *                 bucket5
+ *                 /      \
+ *             file12     dir6
+ *                     /    \
+ *                 file13    dir7
+ *                          /
+ *                       file14
+ *
+ * + *

Every key is written with RATIS/THREE replication so that + * {@code sizeWithReplica} (3x) is genuinely distinct from the unreplicated + * {@code size}. FSO and Legacy differ only in the OM DB key scheme, exposed + * through the {@code writeDir}/{@code writeKey}/{@code writeReplicatedKey} + * hooks; all constants and assertions are shared here. + */ +public abstract class AbstractTreeNSSummaryScenario extends NSSummaryTestScenario { + + // object names + static final String VOL = "vol"; + static final String VOL_TWO = "vol2"; + static final String VOL_THREE = "vol3"; + static final String BUCKET_ONE = "bucket1"; + static final String BUCKET_TWO = "bucket2"; + static final String BUCKET_THREE = "bucket3"; + static final String BUCKET_FOUR = "bucket4"; + static final String BUCKET_FIVE = "bucket5"; + static final String KEY_ONE = "file1"; + static final String KEY_TWO = "dir1/dir2/file2"; + static final String KEY_THREE = "dir1/dir3/file3"; + static final String KEY_FOUR = "file4"; + static final String KEY_FIVE = "file5"; + static final String KEY_SIX = "dir1/dir4/file6"; + static final String KEY_SEVEN = "dir1/file7"; + static final String KEY_EIGHT = "file8"; + static final String KEY_NINE = "dir5/file9"; + static final String KEY_TEN = "dir5/file10"; + static final String KEY_ELEVEN = "file11"; + static final String KEY_TWELVE = "file12"; + static final String KEY_THIRTEEN = "dir6/file13"; + static final String KEY_FOURTEEN = "dir6/dir7/file14"; + static final String FILE_ONE = "file1"; + static final String FILE_TWO = "file2"; + static final String FILE_THREE = "file3"; + static final String FILE_FOUR = "file4"; + static final String FILE_FIVE = "file5"; + static final String FILE_SIX = "file6"; + static final String FILE_SEVEN = "file7"; + static final String FILE_EIGHT = "file8"; + static final String FILE_NINE = "file9"; + static final String FILE_TEN = "file10"; + static final String FILE_ELEVEN = "file11"; + static final String FILE_TWELVE = "file12"; + static final String FILE_THIRTEEN = "file13"; + static final String FILE_FOURTEEN = "file14"; + static final String DIR_ONE = "dir1"; + static final String DIR_TWO = "dir2"; + static final String DIR_THREE = "dir3"; + static final String DIR_FOUR = "dir4"; + static final String DIR_FIVE = "dir5"; + static final String DIR_SIX = "dir6"; + static final String DIR_SEVEN = "dir7"; + + // object IDs + static final long VOL_OBJECT_ID = 0L; + static final long BUCKET_ONE_OBJECT_ID = 1L; + static final long BUCKET_TWO_OBJECT_ID = 2L; + static final long KEY_ONE_OBJECT_ID = 3L; + static final long DIR_ONE_OBJECT_ID = 4L; + static final long KEY_TWO_OBJECT_ID = 5L; + static final long KEY_FOUR_OBJECT_ID = 6L; + static final long DIR_TWO_OBJECT_ID = 7L; + static final long KEY_THREE_OBJECT_ID = 8L; + static final long KEY_FIVE_OBJECT_ID = 9L; + static final long KEY_SIX_OBJECT_ID = 10L; + static final long DIR_THREE_OBJECT_ID = 11L; + static final long DIR_FOUR_OBJECT_ID = 12L; + static final long KEY_SEVEN_OBJECT_ID = 13L; + static final long VOL_TWO_OBJECT_ID = 14L; + static final long BUCKET_THREE_OBJECT_ID = 15L; + static final long BUCKET_FOUR_OBJECT_ID = 16L; + static final long KEY_EIGHT_OBJECT_ID = 17L; + static final long DIR_FIVE_OBJECT_ID = 18L; + static final long KEY_NINE_OBJECT_ID = 19L; + static final long KEY_TEN_OBJECT_ID = 20L; + static final long KEY_ELEVEN_OBJECT_ID = 21L; + static final long VOL_THREE_OBJECT_ID = 22L; + static final long DIR_SIX_OBJECT_ID = 23L; + static final long DIR_SEVEN_OBJECT_ID = 24L; + static final long FILE_TWELVE_OBJECT_ID = 25L; + static final long FILE_THIRTEEN_OBJECT_ID = 26L; + static final long FILE_FOURTEEN_OBJECT_ID = 27L; + static final long BUCKET_FIVE_OBJECT_ID = 28L; + + // data size in bytes (unreplicated) + static final long KEY_ONE_SIZE = 500L; // bin 0 + static final long KEY_TWO_SIZE = OzoneConsts.KB + 1; // bin 1 + static final long KEY_THREE_SIZE = 4 * OzoneConsts.KB + 1; // bin 3 + static final long KEY_FOUR_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 + static final long KEY_FIVE_SIZE = 100L; // bin 0 + static final long KEY_SIX_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 + static final long KEY_SEVEN_SIZE = 4 * OzoneConsts.KB + 1; + static final long KEY_EIGHT_SIZE = OzoneConsts.KB + 1; // bin 1 + static final long KEY_NINE_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 + static final long KEY_TEN_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 + static final long KEY_ELEVEN_SIZE = OzoneConsts.KB + 1; // bin 1 + static final long KEY_TWELVE_SIZE = OzoneConsts.KB; + static final long KEY_THIRTEEN_SIZE = OzoneConsts.KB; + static final long KEY_FOURTEEN_SIZE = OzoneConsts.KB; + + // replicated sizes (RATIS/THREE => 3x the unreplicated size) + static final long FILE1_SIZE_WITH_REPLICA = repl(KEY_ONE_SIZE); + static final long FILE2_SIZE_WITH_REPLICA = repl(KEY_TWO_SIZE); + static final long FILE3_SIZE_WITH_REPLICA = repl(KEY_THREE_SIZE); + static final long FILE4_SIZE_WITH_REPLICA = repl(KEY_FOUR_SIZE); + static final long FILE5_SIZE_WITH_REPLICA = repl(KEY_FIVE_SIZE); + static final long FILE6_SIZE_WITH_REPLICA = repl(KEY_SIX_SIZE); + static final long FILE7_SIZE_WITH_REPLICA = repl(KEY_SEVEN_SIZE); + static final long FILE8_SIZE_WITH_REPLICA = repl(KEY_EIGHT_SIZE); + static final long FILE9_SIZE_WITH_REPLICA = repl(KEY_NINE_SIZE); + static final long FILE10_SIZE_WITH_REPLICA = repl(KEY_TEN_SIZE); + static final long FILE11_SIZE_WITH_REPLICA = repl(KEY_ELEVEN_SIZE); + static final long FILE12_SIZE_WITH_REPLICA = repl(KEY_TWELVE_SIZE); + static final long FILE13_SIZE_WITH_REPLICA = repl(KEY_THIRTEEN_SIZE); + static final long FILE14_SIZE_WITH_REPLICA = repl(KEY_FOURTEEN_SIZE); + + static final long REPLICA_UNDER_ROOT = + FILE1_SIZE_WITH_REPLICA + FILE2_SIZE_WITH_REPLICA + FILE3_SIZE_WITH_REPLICA + + FILE4_SIZE_WITH_REPLICA + FILE5_SIZE_WITH_REPLICA + FILE6_SIZE_WITH_REPLICA + + FILE7_SIZE_WITH_REPLICA + FILE8_SIZE_WITH_REPLICA + FILE9_SIZE_WITH_REPLICA + + FILE10_SIZE_WITH_REPLICA + FILE11_SIZE_WITH_REPLICA + FILE12_SIZE_WITH_REPLICA + + FILE13_SIZE_WITH_REPLICA + FILE14_SIZE_WITH_REPLICA; + static final long REPLICA_UNDER_VOL = + FILE1_SIZE_WITH_REPLICA + FILE2_SIZE_WITH_REPLICA + FILE3_SIZE_WITH_REPLICA + + FILE4_SIZE_WITH_REPLICA + FILE5_SIZE_WITH_REPLICA + FILE6_SIZE_WITH_REPLICA + + FILE7_SIZE_WITH_REPLICA; + static final long REPLICA_UNDER_VOL_TWO = + FILE8_SIZE_WITH_REPLICA + FILE9_SIZE_WITH_REPLICA + FILE10_SIZE_WITH_REPLICA + + FILE11_SIZE_WITH_REPLICA; + static final long REPLICA_UNDER_VOL_THREE = + FILE12_SIZE_WITH_REPLICA + FILE13_SIZE_WITH_REPLICA + FILE14_SIZE_WITH_REPLICA; + static final long REPLICA_UNDER_BUCKET1 = + FILE1_SIZE_WITH_REPLICA + FILE2_SIZE_WITH_REPLICA + FILE3_SIZE_WITH_REPLICA + + FILE6_SIZE_WITH_REPLICA + FILE7_SIZE_WITH_REPLICA; + static final long REPLICA_UNDER_BUCKET2 = + FILE4_SIZE_WITH_REPLICA + FILE5_SIZE_WITH_REPLICA; + static final long REPLICA_UNDER_DIR1 = + FILE2_SIZE_WITH_REPLICA + FILE3_SIZE_WITH_REPLICA + FILE6_SIZE_WITH_REPLICA + + FILE7_SIZE_WITH_REPLICA; + static final long REPLICA_UNDER_DIR2 = FILE2_SIZE_WITH_REPLICA; + static final long REPLICA_UNDER_KEY = FILE4_SIZE_WITH_REPLICA; + static final long REPLICA_UNDER_DIR6 = + FILE13_SIZE_WITH_REPLICA + FILE14_SIZE_WITH_REPLICA; + + // quota in bytes + static final long ROOT_QUOTA = 2 * (2 * OzoneConsts.MB); + static final long VOL_QUOTA = 2 * OzoneConsts.MB; + static final long VOL_TWO_QUOTA = 2 * OzoneConsts.MB; + static final long VOL_THREE_QUOTA = 2 * OzoneConsts.MB; + static final long BUCKET_ONE_QUOTA = OzoneConsts.MB; + static final long BUCKET_TWO_QUOTA = OzoneConsts.MB; + static final long BUCKET_THREE_QUOTA = OzoneConsts.MB; + static final long BUCKET_FOUR_QUOTA = OzoneConsts.MB; + static final long BUCKET_FIVE_QUOTA = OzoneConsts.MB; + + // request paths + static final String VOL_PATH = "/vol"; + static final String VOL_TWO_PATH = "/vol2"; + static final String VOL_THREE_PATH = "/vol3"; + static final String BUCKET_ONE_PATH = "/vol/bucket1"; + static final String BUCKET_TWO_PATH = "/vol/bucket2"; + static final String DIR_ONE_PATH = "/vol/bucket1/dir1"; + static final String DIR_TWO_PATH = "/vol/bucket1/dir1/dir2"; + static final String DIR_THREE_PATH = "/vol/bucket1/dir1/dir3"; + static final String DIR_FOUR_PATH = "/vol/bucket1/dir1/dir4"; + static final String MULTI_BLOCK_KEY_PATH = "/vol/bucket1/dir1/file7"; + + // unreplicated (namespace) data sizes used by non-replica DU and quota + static final long ROOT_DATA_SIZE = REPLICA_UNDER_ROOT; + static final long VOL_DATA_SIZE = KEY_ONE_SIZE + KEY_TWO_SIZE + KEY_THREE_SIZE + + KEY_FOUR_SIZE + KEY_FIVE_SIZE + KEY_SIX_SIZE + KEY_SEVEN_SIZE; + static final long VOL_TWO_DATA_SIZE = + KEY_EIGHT_SIZE + KEY_NINE_SIZE + KEY_TEN_SIZE + KEY_ELEVEN_SIZE; + static final long BUCKET_ONE_DATA_SIZE = KEY_ONE_SIZE + KEY_TWO_SIZE + + KEY_THREE_SIZE + KEY_SIX_SIZE + KEY_SEVEN_SIZE; + static final long BUCKET_TWO_DATA_SIZE = KEY_FOUR_SIZE + KEY_FIVE_SIZE; + static final long DIR_ONE_DATA_SIZE = + KEY_TWO_SIZE + KEY_THREE_SIZE + KEY_SIX_SIZE + KEY_SEVEN_SIZE; + + protected AbstractTreeNSSummaryScenario(String displayName) { + super(displayName); + } + + private static long repl(long size) { + return getReplicatedSize(size, RatisReplicationConfig.getInstance(THREE)); + } + + abstract BucketLayout getBucketLayout(); + + /** FSO leaves this alone; Legacy enables filesystem paths. */ + abstract void configureOmConfiguration(OzoneConfiguration conf); + + abstract void reprocess(ReconNamespaceSummaryManager reconNamespaceSummaryManager, + ReconOMMetadataManager reconOMMetadataManager, OzoneConfiguration conf) + throws IOException; + + /** + * Write a directory. {@code legacyPath} and {@code fsoParentObjectId} let the + * FSO (directory table, real parent) and Legacy (key table, path-based) + * schemes coexist behind one call site. + */ + @SuppressWarnings("checkstyle:ParameterNumber") + abstract void writeDir(ReconOMMetadataManager recon, String dirName, + String legacyPath, long dirObjectId, long fsoParentObjectId, + String bucketName, String volName, long bucketObjectId, long volObjectId) + throws IOException; + + @SuppressWarnings("checkstyle:ParameterNumber") + abstract void writeReplicatedKey(ReconOMMetadataManager recon, String key, + String bucket, String vol, String fileName, long keyObjectId, + long fsoParentObjectId, long bucketObjectId, long volObjectId, + OmKeyLocationInfoGroup locationGroup, long dataSize) throws IOException; + + @Override + boolean hasDirectories() { + return true; + } + + @Override + boolean hasVolumeThree() { + return true; + } + + @Override + long rootQuota() { + return ROOT_QUOTA; + } + + @Override + long rootDataSize() { + return ROOT_DATA_SIZE; + } + + @Override + OMMetadataManager initializeOmMetadataManager(File omDbDir, + OzoneConfiguration conf) throws IOException { + conf.set(OZONE_OM_DB_DIRS, omDbDir.getAbsolutePath()); + configureOmConfiguration(conf); + OMMetadataManager omMetadataManager = new OmMetadataManagerImpl(conf, null); + + putVolume(omMetadataManager, VOL, VOL_OBJECT_ID, VOL_QUOTA); + putVolume(omMetadataManager, VOL_TWO, VOL_TWO_OBJECT_ID, VOL_TWO_QUOTA); + putVolume(omMetadataManager, VOL_THREE, VOL_THREE_OBJECT_ID, VOL_THREE_QUOTA); + + putBucket(omMetadataManager, VOL, BUCKET_ONE, BUCKET_ONE_OBJECT_ID, + BUCKET_ONE_QUOTA); + putBucket(omMetadataManager, VOL, BUCKET_TWO, BUCKET_TWO_OBJECT_ID, + BUCKET_TWO_QUOTA); + putBucket(omMetadataManager, VOL_TWO, BUCKET_THREE, BUCKET_THREE_OBJECT_ID, + BUCKET_THREE_QUOTA); + putBucket(omMetadataManager, VOL_TWO, BUCKET_FOUR, BUCKET_FOUR_OBJECT_ID, + BUCKET_FOUR_QUOTA); + putBucket(omMetadataManager, VOL_THREE, BUCKET_FIVE, BUCKET_FIVE_OBJECT_ID, + BUCKET_FIVE_QUOTA); + return omMetadataManager; + } + + private void putVolume(OMMetadataManager omMetadataManager, String volume, + long objectId, long quota) throws IOException { + omMetadataManager.getVolumeTable().put( + omMetadataManager.getVolumeKey(volume), + OmVolumeArgs.newBuilder() + .setObjectID(objectId) + .setVolume(volume) + .setAdminName(TEST_USER) + .setOwnerName(TEST_USER) + .setQuotaInBytes(quota) + .build()); + } + + private void putBucket(OMMetadataManager omMetadataManager, String volume, + String bucket, long objectId, long quota) throws IOException { + OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() + .setVolumeName(volume) + .setBucketName(bucket) + .setObjectID(objectId) + .setQuotaInBytes(quota) + .setBucketLayout(getBucketLayout()) + .build(); + omMetadataManager.getBucketTable().put( + omMetadataManager.getBucketKey(volume, bucket), bucketInfo); + } + + @Override + void populateAndReprocess( + ReconNamespaceSummaryManager reconNamespaceSummaryManager, + ReconOMMetadataManager recon, OzoneConfiguration conf) throws Exception { + writeDirectories(recon); + writeVolumeThree(recon); + writeReplicatedKeys(recon); + reprocess(reconNamespaceSummaryManager, recon, conf); + } + + private void writeDirectories(ReconOMMetadataManager recon) + throws IOException { + String p = OzoneConsts.OM_KEY_PREFIX; + writeDir(recon, DIR_ONE, DIR_ONE + p, DIR_ONE_OBJECT_ID, + BUCKET_ONE_OBJECT_ID, BUCKET_ONE, VOL, BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID); + writeDir(recon, DIR_TWO, DIR_ONE + p + DIR_TWO + p, DIR_TWO_OBJECT_ID, + DIR_ONE_OBJECT_ID, BUCKET_ONE, VOL, BUCKET_ONE_OBJECT_ID, VOL_OBJECT_ID); + writeDir(recon, DIR_THREE, DIR_ONE + p + DIR_THREE + p, DIR_THREE_OBJECT_ID, + DIR_ONE_OBJECT_ID, BUCKET_ONE, VOL, BUCKET_ONE_OBJECT_ID, VOL_OBJECT_ID); + writeDir(recon, DIR_FOUR, DIR_ONE + p + DIR_FOUR + p, DIR_FOUR_OBJECT_ID, + DIR_ONE_OBJECT_ID, BUCKET_ONE, VOL, BUCKET_ONE_OBJECT_ID, VOL_OBJECT_ID); + writeDir(recon, DIR_FIVE, DIR_FIVE + p, DIR_FIVE_OBJECT_ID, + BUCKET_THREE_OBJECT_ID, BUCKET_THREE, VOL_TWO, BUCKET_THREE_OBJECT_ID, + VOL_TWO_OBJECT_ID); + } + + private void writeVolumeThree(ReconOMMetadataManager recon) + throws IOException { + String p = OzoneConsts.OM_KEY_PREFIX; + writeDir(recon, DIR_SIX, DIR_SIX + p, DIR_SIX_OBJECT_ID, + BUCKET_FIVE_OBJECT_ID, BUCKET_FIVE, VOL_THREE, BUCKET_FIVE_OBJECT_ID, + VOL_THREE_OBJECT_ID); + writeDir(recon, DIR_SEVEN, DIR_SIX + p + DIR_SEVEN + p, DIR_SEVEN_OBJECT_ID, + DIR_SIX_OBJECT_ID, BUCKET_FIVE, VOL_THREE, BUCKET_FIVE_OBJECT_ID, + VOL_THREE_OBJECT_ID); + // vol3 keys use the same (object-id parented) call for FSO and Legacy. + writeRatisKey(recon, KEY_TWELVE, BUCKET_FIVE, VOL_THREE, FILE_TWELVE, + FILE_TWELVE_OBJECT_ID, BUCKET_FIVE_OBJECT_ID, BUCKET_FIVE_OBJECT_ID, + VOL_THREE_OBJECT_ID, KEY_TWELVE_SIZE); + writeRatisKey(recon, KEY_THIRTEEN, BUCKET_FIVE, VOL_THREE, FILE_THIRTEEN, + FILE_THIRTEEN_OBJECT_ID, DIR_SIX_OBJECT_ID, BUCKET_FIVE_OBJECT_ID, + VOL_THREE_OBJECT_ID, KEY_THIRTEEN_SIZE); + writeRatisKey(recon, KEY_FOURTEEN, BUCKET_FIVE, VOL_THREE, FILE_FOURTEEN, + FILE_FOURTEEN_OBJECT_ID, DIR_SEVEN_OBJECT_ID, BUCKET_FIVE_OBJECT_ID, + VOL_THREE_OBJECT_ID, KEY_FOURTEEN_SIZE); + } + + @SuppressWarnings("checkstyle:ParameterNumber") + private void writeRatisKey(ReconOMMetadataManager recon, String key, + String bucket, String vol, String fileName, long objectId, + long parentObjectId, long bucketObjectId, long volObjectId, long dataSize) + throws IOException { + writeKeyToOm(recon, key, bucket, vol, fileName, objectId, parentObjectId, + bucketObjectId, volObjectId, dataSize, getBucketLayout(), + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE), + org.apache.hadoop.util.Time.now(), true); + } + + private void writeReplicatedKeys(ReconOMMetadataManager recon) + throws IOException { + OmKeyLocationInfoGroup group1 = + NSSummaryEndpointTestBase.getLocationInfoGroup1(); + OmKeyLocationInfoGroup group2 = + NSSummaryEndpointTestBase.getLocationInfoGroup2(); + writeReplicatedKey(recon, KEY_ONE, BUCKET_ONE, VOL, FILE_ONE, + KEY_ONE_OBJECT_ID, BUCKET_ONE_OBJECT_ID, BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, group1, KEY_ONE_SIZE); + writeReplicatedKey(recon, KEY_TWO, BUCKET_ONE, VOL, FILE_TWO, + KEY_TWO_OBJECT_ID, DIR_TWO_OBJECT_ID, BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, group2, KEY_TWO_SIZE); + writeReplicatedKey(recon, KEY_THREE, BUCKET_ONE, VOL, FILE_THREE, + KEY_THREE_OBJECT_ID, DIR_THREE_OBJECT_ID, BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, group1, KEY_THREE_SIZE); + writeReplicatedKey(recon, KEY_FOUR, BUCKET_TWO, VOL, FILE_FOUR, + KEY_FOUR_OBJECT_ID, BUCKET_TWO_OBJECT_ID, BUCKET_TWO_OBJECT_ID, + VOL_OBJECT_ID, group2, KEY_FOUR_SIZE); + writeReplicatedKey(recon, KEY_FIVE, BUCKET_TWO, VOL, FILE_FIVE, + KEY_FIVE_OBJECT_ID, BUCKET_TWO_OBJECT_ID, BUCKET_TWO_OBJECT_ID, + VOL_OBJECT_ID, group1, KEY_FIVE_SIZE); + writeReplicatedKey(recon, KEY_SIX, BUCKET_ONE, VOL, FILE_SIX, + KEY_SIX_OBJECT_ID, DIR_FOUR_OBJECT_ID, BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, group2, KEY_SIX_SIZE); + writeReplicatedKey(recon, KEY_SEVEN, BUCKET_ONE, VOL, FILE_SEVEN, + KEY_SEVEN_OBJECT_ID, DIR_ONE_OBJECT_ID, BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, group1, KEY_SEVEN_SIZE); + writeReplicatedKey(recon, KEY_EIGHT, BUCKET_THREE, VOL_TWO, FILE_EIGHT, + KEY_EIGHT_OBJECT_ID, BUCKET_THREE_OBJECT_ID, BUCKET_THREE_OBJECT_ID, + VOL_TWO_OBJECT_ID, group2, KEY_EIGHT_SIZE); + writeReplicatedKey(recon, KEY_NINE, BUCKET_THREE, VOL_TWO, FILE_NINE, + KEY_NINE_OBJECT_ID, DIR_FIVE_OBJECT_ID, BUCKET_THREE_OBJECT_ID, + VOL_TWO_OBJECT_ID, group1, KEY_NINE_SIZE); + writeReplicatedKey(recon, KEY_TEN, BUCKET_THREE, VOL_TWO, FILE_TEN, + KEY_TEN_OBJECT_ID, DIR_FIVE_OBJECT_ID, BUCKET_THREE_OBJECT_ID, + VOL_TWO_OBJECT_ID, group2, KEY_TEN_SIZE); + writeReplicatedKey(recon, KEY_ELEVEN, BUCKET_FOUR, VOL_TWO, FILE_ELEVEN, + KEY_ELEVEN_OBJECT_ID, BUCKET_FOUR_OBJECT_ID, BUCKET_FOUR_OBJECT_ID, + VOL_TWO_OBJECT_ID, group1, KEY_ELEVEN_SIZE); + } + + @Override + void writeMultiBlockKey(ReconOMMetadataManager recon) throws IOException { + writeReplicatedKey(recon, KEY_SEVEN, BUCKET_ONE, VOL, FILE_SEVEN, + KEY_SEVEN_OBJECT_ID, DIR_ONE_OBJECT_ID, BUCKET_ONE_OBJECT_ID, + VOL_OBJECT_ID, NSSummaryEndpointTestBase.getLocationInfoGroup1(), + KEY_SEVEN_SIZE); + } + + @Override + String multiBlockKeyPath() { + return MULTI_BLOCK_KEY_PATH; + } + + @Override + long multiBlockKeyReplicatedSize() { + return FILE7_SIZE_WITH_REPLICA; + } + + // --------------------------------------------------------------------------- + // Assertions + // --------------------------------------------------------------------------- + + @Override + void assertBasicInfoRoot(NSSummaryEndpoint endpoint, + ReconOMMetadataManager recon) throws Exception { + String username = "myuser"; + OmPrefixInfo omPrefixInfo = OmPrefixInfo.newBuilder() + .setName(ROOT_PATH) + .setObjectID(10) + .setUpdateID(100) + .setAcls(singletonList(OzoneAcl.of(USER, username, ACCESS, WRITE))) + .addAllMetadata(singletonMap("key", "value")) + .build(); + recon.getPrefixTable().put(OzoneConsts.OM_KEY_PREFIX, omPrefixInfo); + + NamespaceSummaryResponse root = (NamespaceSummaryResponse) + endpoint.getBasicInfo(ROOT_PATH).getEntity(); + assertEquals(EntityType.ROOT, root.getEntityType()); + assertEquals(3, root.getCountStats().getNumVolume()); + assertEquals(5, root.getCountStats().getNumBucket()); + assertEquals(7, root.getCountStats().getNumTotalDir()); + assertEquals(14, root.getCountStats().getNumTotalKey()); + assertEquals("USER", root.getObjectDBInfo().getAcls().get(0).getType()); + assertEquals("WRITE", + root.getObjectDBInfo().getAcls().get(0).getAclList().get(0)); + assertEquals(username, root.getObjectDBInfo().getAcls().get(0).getName()); + assertEquals("value", root.getObjectDBInfo().getMetadata().get("key")); + assertEquals("ACCESS", root.getObjectDBInfo().getAcls().get(0).getScope()); + } + + @Override + void assertBasicInfoVolume(NSSummaryEndpoint endpoint) throws Exception { + NamespaceSummaryResponse vol = (NamespaceSummaryResponse) + endpoint.getBasicInfo(VOL_PATH).getEntity(); + assertEquals(EntityType.VOLUME, vol.getEntityType()); + assertEquals(2, vol.getCountStats().getNumBucket()); + assertEquals(4, vol.getCountStats().getNumTotalDir()); + assertEquals(7, vol.getCountStats().getNumTotalKey()); + assertEquals(TEST_USER, + ((VolumeObjectDBInfo) vol.getObjectDBInfo()).getAdmin()); + assertEquals(TEST_USER, + ((VolumeObjectDBInfo) vol.getObjectDBInfo()).getOwner()); + assertEquals(VOL, vol.getObjectDBInfo().getName()); + assertEquals(2097152, vol.getObjectDBInfo().getQuotaInBytes()); + assertEquals(-1, vol.getObjectDBInfo().getQuotaInNamespace()); + } + + @Override + void assertBasicInfoBucketOne(NSSummaryEndpoint endpoint) throws Exception { + NamespaceSummaryResponse bucket = (NamespaceSummaryResponse) + endpoint.getBasicInfo(BUCKET_ONE_PATH).getEntity(); + assertEquals(EntityType.BUCKET, bucket.getEntityType()); + assertEquals(4, bucket.getCountStats().getNumTotalDir()); + assertEquals(5, bucket.getCountStats().getNumTotalKey()); + assertEquals(VOL, + ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getVolumeName()); + assertEquals(StorageType.DISK, + ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getStorageType()); + assertEquals(getBucketLayout(), + ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getBucketLayout()); + assertEquals(BUCKET_ONE, + ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getName()); + } + + @Override + void assertBasicInfoBucketTwo(NSSummaryEndpoint endpoint) throws Exception { + NamespaceSummaryResponse bucket = (NamespaceSummaryResponse) + endpoint.getBasicInfo(BUCKET_TWO_PATH).getEntity(); + assertEquals(EntityType.BUCKET, bucket.getEntityType()); + assertEquals(0, bucket.getCountStats().getNumTotalDir()); + assertEquals(2, bucket.getCountStats().getNumTotalKey()); + assertEquals(VOL, + ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getVolumeName()); + assertEquals(StorageType.DISK, + ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getStorageType()); + assertEquals(getBucketLayout(), + ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getBucketLayout()); + assertEquals(BUCKET_TWO, + ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getName()); + } + + /** Directory basic info: identical for FSO and Legacy. */ + void assertBasicInfoDir(NSSummaryEndpoint endpoint) throws Exception { + NamespaceSummaryResponse dir = (NamespaceSummaryResponse) + endpoint.getBasicInfo(DIR_ONE_PATH).getEntity(); + assertEquals(EntityType.DIRECTORY, dir.getEntityType()); + assertEquals(3, dir.getCountStats().getNumTotalDir()); + assertEquals(4, dir.getCountStats().getNumTotalKey()); + assertEquals(DIR_ONE, dir.getObjectDBInfo().getName()); + assertEquals(0, dir.getObjectDBInfo().getMetadata().size()); + assertEquals(0, dir.getObjectDBInfo().getQuotaInBytes()); + assertEquals(0, dir.getObjectDBInfo().getQuotaInNamespace()); + assertEquals(0, dir.getObjectDBInfo().getUsedNamespace()); + } + + @Override + void assertDiskUsageRoot(NSSummaryEndpoint endpoint) throws Exception { + DUResponse duRootRes = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, ROOT_PATH, false); + assertEquals(3, duRootRes.getCount()); + List duData = duRootRes.getDuData(); + Collections.sort(duData, + Comparator.comparing(DUResponse.DiskUsage::getSubpath)); + assertEquals(VOL_PATH, duData.get(0).getSubpath()); + assertEquals(VOL_TWO_PATH, duData.get(1).getSubpath()); + assertEquals(VOL_DATA_SIZE, duData.get(0).getSize()); + assertEquals(VOL_TWO_DATA_SIZE, duData.get(1).getSize()); + } + + @Override + void assertDiskUsageVolume(NSSummaryEndpoint endpoint) throws Exception { + DUResponse duVolRes = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, VOL_PATH, false); + assertEquals(2, duVolRes.getCount()); + List duData = duVolRes.getDuData(); + Collections.sort(duData, + Comparator.comparing(DUResponse.DiskUsage::getSubpath)); + assertEquals(BUCKET_ONE_PATH, duData.get(0).getSubpath()); + assertEquals(BUCKET_TWO_PATH, duData.get(1).getSubpath()); + assertEquals(BUCKET_ONE_DATA_SIZE, duData.get(0).getSize()); + assertEquals(BUCKET_TWO_DATA_SIZE, duData.get(1).getSize()); + } + + /** Bucket-level DU (tree-only): bucket1 -> dir1. */ + void assertDiskUsageBucket(NSSummaryEndpoint endpoint) throws Exception { + DUResponse duBucketRes = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, BUCKET_ONE_PATH, false); + assertEquals(1, duBucketRes.getCount()); + DUResponse.DiskUsage duDir1 = duBucketRes.getDuData().get(0); + assertEquals(DIR_ONE_PATH, duDir1.getSubpath()); + assertEquals(DIR_ONE_DATA_SIZE, duDir1.getSize()); + } + + /** Directory-level DU (tree-only): dir1 -> dir2, dir3, dir4. */ + void assertDiskUsageDir(NSSummaryEndpoint endpoint) throws Exception { + DUResponse duDirRes = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, DIR_ONE_PATH, false); + assertEquals(3, duDirRes.getCount()); + List duSubDir = duDirRes.getDuData(); + Collections.sort(duSubDir, + Comparator.comparing(DUResponse.DiskUsage::getSubpath)); + assertEquals(DIR_TWO_PATH, duSubDir.get(0).getSubpath()); + assertEquals(KEY_TWO_SIZE, duSubDir.get(0).getSize()); + assertEquals(DIR_THREE_PATH, duSubDir.get(1).getSubpath()); + assertEquals(KEY_THREE_SIZE, duSubDir.get(1).getSize()); + assertEquals(DIR_FOUR_PATH, duSubDir.get(2).getSubpath()); + assertEquals(KEY_SIX_SIZE, duSubDir.get(2).getSize()); + } + + @Override + void assertDiskUsageKey(NSSummaryEndpoint endpoint) throws Exception { + DUResponse keyObj = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, KEY_PATH, false); + assertEquals(0, keyObj.getCount()); + assertEquals(KEY_FOUR_SIZE, keyObj.getSize()); + } + + @Override + void assertQuotaUsage(NSSummaryEndpoint endpoint) throws Exception { + QuotaUsageResponse quRootRes = (QuotaUsageResponse) + endpoint.getQuotaUsage(ROOT_PATH).getEntity(); + assertEquals(ROOT_QUOTA, quRootRes.getQuota()); + assertEquals(ROOT_DATA_SIZE, quRootRes.getQuotaUsed()); + + QuotaUsageResponse quVolRes = (QuotaUsageResponse) + endpoint.getQuotaUsage(VOL_PATH).getEntity(); + assertEquals(VOL_QUOTA, quVolRes.getQuota()); + assertEquals(VOL_DATA_SIZE, quVolRes.getQuotaUsed()); + + QuotaUsageResponse quBucketRes = (QuotaUsageResponse) + endpoint.getQuotaUsage(BUCKET_ONE_PATH).getEntity(); + assertEquals(BUCKET_ONE_QUOTA, quBucketRes.getQuota()); + assertEquals(BUCKET_ONE_DATA_SIZE, quBucketRes.getQuotaUsed()); + + QuotaUsageResponse quBucketRes2 = (QuotaUsageResponse) + endpoint.getQuotaUsage(BUCKET_TWO_PATH).getEntity(); + assertEquals(BUCKET_TWO_QUOTA, quBucketRes2.getQuota()); + assertEquals(BUCKET_TWO_DATA_SIZE, quBucketRes2.getQuotaUsed()); + + QuotaUsageResponse naDir = (QuotaUsageResponse) + endpoint.getQuotaUsage(DIR_ONE_PATH).getEntity(); + assertEquals(ResponseStatus.TYPE_NOT_APPLICABLE, naDir.getResponseCode()); + + QuotaUsageResponse naKey = (QuotaUsageResponse) + endpoint.getQuotaUsage(KEY_PATH).getEntity(); + assertEquals(ResponseStatus.TYPE_NOT_APPLICABLE, naKey.getResponseCode()); + + QuotaUsageResponse invalid = (QuotaUsageResponse) + endpoint.getQuotaUsage(INVALID_PATH).getEntity(); + assertEquals(ResponseStatus.PATH_NOT_FOUND, invalid.getResponseCode()); + } + + @Override + void assertFileSizeDist(NSSummaryEndpoint endpoint) throws Exception { + NSSummaryEndpointTestBase.checkFileSizeDist(endpoint, ROOT_PATH, 5, 3, 4, 2); + NSSummaryEndpointTestBase.checkFileSizeDist(endpoint, VOL_PATH, 2, 1, 2, 2); + NSSummaryEndpointTestBase.checkFileSizeDist( + endpoint, BUCKET_ONE_PATH, 1, 1, 1, 2); + NSSummaryEndpointTestBase.checkFileSizeDist( + endpoint, DIR_ONE_PATH, 0, 1, 1, 2); + } + + @Override + void assertDataSizeUnderRootWithReplication(NSSummaryEndpoint endpoint) + throws IOException { + DUResponse res = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, ROOT_PATH, true); + assertEquals(ResponseStatus.OK, res.getStatus()); + assertEquals(REPLICA_UNDER_ROOT, res.getSizeWithReplica()); + assertEquals(REPLICA_UNDER_VOL, res.getDuData().get(0).getSizeWithReplica()); + } + + @Override + void assertDataSizeUnderVolWithReplication(NSSummaryEndpoint endpoint) + throws IOException { + DUResponse res = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, VOL_PATH, true); + assertEquals(ResponseStatus.OK, res.getStatus()); + assertEquals(REPLICA_UNDER_VOL, res.getSizeWithReplica()); + assertEquals(REPLICA_UNDER_BUCKET1, + res.getDuData().get(0).getSizeWithReplica()); + } + + /** Bucket-level replicated DU (tree-only). */ + void assertDataSizeUnderBucketWithReplication(NSSummaryEndpoint endpoint) + throws IOException { + DUResponse res = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, BUCKET_ONE_PATH, true); + assertEquals(ResponseStatus.OK, res.getStatus()); + assertEquals(REPLICA_UNDER_BUCKET1, res.getSizeWithReplica()); + assertEquals(REPLICA_UNDER_DIR1, + res.getDuData().get(0).getSizeWithReplica()); + } + + /** Directory-level replicated DU (tree-only): dir1 -> dir2. */ + void assertDataSizeUnderDirWithReplication(NSSummaryEndpoint endpoint) + throws IOException { + DUResponse res = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, DIR_ONE_PATH, true); + assertEquals(ResponseStatus.OK, res.getStatus()); + assertEquals(REPLICA_UNDER_DIR1, res.getSizeWithReplica()); + assertEquals(REPLICA_UNDER_DIR2, + res.getDuData().get(0).getSizeWithReplica()); + } + + @Override + void assertDataSizeUnderKeyWithReplication(NSSummaryEndpoint endpoint) + throws IOException { + DUResponse res = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, KEY_PATH, true); + assertEquals(ResponseStatus.OK, res.getStatus()); + assertEquals(REPLICA_UNDER_KEY, res.getSizeWithReplica()); + } + + /** + * Replicated size propagates from files up through the directory hierarchy + * (tree-only). With RATIS/THREE keys, each level's replicated size is 3x its + * unreplicated size. + */ + void assertReplicatedSizePropagation(NSSummaryEndpoint endpoint) + throws IOException { + assertEquals(FILE2_SIZE_WITH_REPLICA, replicaSize(endpoint, DIR_TWO_PATH + "/file2")); + assertEquals(FILE3_SIZE_WITH_REPLICA, replicaSize(endpoint, DIR_THREE_PATH + "/file3")); + assertEquals(FILE6_SIZE_WITH_REPLICA, replicaSize(endpoint, DIR_FOUR_PATH + "/file6")); + assertEquals(FILE7_SIZE_WITH_REPLICA, replicaSize(endpoint, DIR_ONE_PATH + "/file7")); + + assertEquals(FILE2_SIZE_WITH_REPLICA, replicaSize(endpoint, DIR_TWO_PATH)); + assertEquals(FILE3_SIZE_WITH_REPLICA, replicaSize(endpoint, DIR_THREE_PATH)); + assertEquals(FILE6_SIZE_WITH_REPLICA, replicaSize(endpoint, DIR_FOUR_PATH)); + assertEquals(REPLICA_UNDER_DIR1, replicaSize(endpoint, DIR_ONE_PATH)); + assertEquals(REPLICA_UNDER_BUCKET1, replicaSize(endpoint, BUCKET_ONE_PATH)); + assertEquals(REPLICA_UNDER_BUCKET2, replicaSize(endpoint, BUCKET_TWO_PATH)); + assertEquals(REPLICA_UNDER_VOL, replicaSize(endpoint, VOL_PATH)); + assertEquals(REPLICA_UNDER_ROOT, replicaSize(endpoint, ROOT_PATH)); + } + + private long replicaSize(NSSummaryEndpoint endpoint, String path) + throws IOException { + return NSSummaryEndpointTestBase.getDiskUsage(endpoint, path, true) + .getSizeWithReplica(); + } + + /** vol3 RATIS DU (tree-only): asserts both the raw and 3x replicated size. */ + void assertRatisReplicationUnderVolumeThree(NSSummaryEndpoint endpoint) + throws IOException { + DUResponse res = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, VOL_THREE_PATH, true); + assertEquals(ResponseStatus.OK, res.getStatus()); + assertEquals(KEY_TWELVE_SIZE + KEY_THIRTEEN_SIZE + KEY_FOURTEEN_SIZE, + res.getSize()); + assertEquals(REPLICA_UNDER_VOL_THREE, res.getSizeWithReplica()); + } + + void assertRatisReplicationUnderBucketFive(NSSummaryEndpoint endpoint) + throws IOException { + DUResponse res = NSSummaryEndpointTestBase.getDiskUsage( + endpoint, VOL_THREE_PATH + "/" + BUCKET_FIVE, true); + assertEquals(ResponseStatus.OK, res.getStatus()); + assertEquals(KEY_TWELVE_SIZE + KEY_THIRTEEN_SIZE + KEY_FOURTEEN_SIZE, + res.getSize()); + assertEquals(REPLICA_UNDER_VOL_THREE, res.getSizeWithReplica()); + } + + void assertRatisReplicationUnderDirSix(NSSummaryEndpoint endpoint) + throws IOException { + DUResponse res = NSSummaryEndpointTestBase.getDiskUsage( + endpoint, VOL_THREE_PATH + "/" + BUCKET_FIVE + "/" + DIR_SIX, true); + assertEquals(ResponseStatus.OK, res.getStatus()); + assertEquals(KEY_THIRTEEN_SIZE + KEY_FOURTEEN_SIZE, res.getSize()); + assertEquals(REPLICA_UNDER_DIR6, res.getSizeWithReplica()); + } +} diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/FlatNSSummaryScenario.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/FlatNSSummaryScenario.java new file mode 100644 index 000000000000..7bd37ab10276 --- /dev/null +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/FlatNSSummaryScenario.java @@ -0,0 +1,655 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.api; + +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS; +import static org.apache.hadoop.ozone.om.helpers.QuotaUtil.getReplicatedSize; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getMockOzoneManagerServiceProvider; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.setConfiguration; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.StorageType; +import org.apache.hadoop.ozone.OmUtils; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.recon.ReconUtils; +import org.apache.hadoop.ozone.recon.api.types.BucketObjectDBInfo; +import org.apache.hadoop.ozone.recon.api.types.DUResponse; +import org.apache.hadoop.ozone.recon.api.types.EntityType; +import org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse; +import org.apache.hadoop.ozone.recon.api.types.QuotaUsageResponse; +import org.apache.hadoop.ozone.recon.api.types.ResponseStatus; +import org.apache.hadoop.ozone.recon.api.types.VolumeObjectDBInfo; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; +import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; +import org.apache.hadoop.ozone.recon.tasks.NSSummaryTaskWithLegacy; +import org.apache.hadoop.ozone.recon.tasks.NSSummaryTaskWithOBS; + +/** + * Flat fixture with OBS buckets (bucket1, bucket2 under vol) and Legacy buckets + * with filesystem paths disabled (bucket3, bucket4 under vol2), so both behave + * as flat object stores: + * 

+ * └── vol
+ *     ├── bucket1 (OBS)   : file1, ////file2, file3///
+ *     └── bucket2 (OBS)   : file4, _//////
+ * └── vol2
+ *     ├── bucket3 (Legacy): file8, //////, ///__file10
+ *     └── bucket4 (Legacy): ////file11
+ *
+ * Every key is written with RATIS/THREE replication. + */ +public class FlatNSSummaryScenario extends NSSummaryTestScenario { + + // object names + private static final String VOL = "vol"; + private static final String VOL_TWO = "vol2"; + private static final String BUCKET_ONE = "bucket1"; + private static final String BUCKET_TWO = "bucket2"; + private static final String BUCKET_THREE = "bucket3"; + private static final String BUCKET_FOUR = "bucket4"; + private static final String KEY_ONE = "file1"; + private static final String KEY_TWO = "////file2"; + private static final String KEY_THREE = "file3///"; + private static final String KEY_FOUR = "file4"; + private static final String KEY_FIVE = "_//////"; + private static final String KEY_EIGHT = "file8"; + private static final String KEY_NINE = "//////"; + private static final String KEY_TEN = "///__file10"; + private static final String KEY_ELEVEN = "////file11"; + private static final String MULTI_BLOCK_FILE = KEY_THREE; + + // object IDs + private static final long VOL_OBJECT_ID = 0L; + private static final long VOL_TWO_OBJECT_ID = 14L; + private static final long BUCKET_ONE_OBJECT_ID = 1L; + private static final long BUCKET_TWO_OBJECT_ID = 2L; + private static final long BUCKET_THREE_OBJECT_ID = 15L; + private static final long BUCKET_FOUR_OBJECT_ID = 16L; + private static final long KEY_ONE_OBJECT_ID = 3L; + private static final long KEY_TWO_OBJECT_ID = 5L; + private static final long KEY_THREE_OBJECT_ID = 8L; + private static final long KEY_FOUR_OBJECT_ID = 6L; + private static final long KEY_FIVE_OBJECT_ID = 9L; + private static final long KEY_EIGHT_OBJECT_ID = 17L; + private static final long KEY_NINE_OBJECT_ID = 19L; + private static final long KEY_TEN_OBJECT_ID = 20L; + private static final long KEY_ELEVEN_OBJECT_ID = 21L; + private static final long MULTI_BLOCK_KEY_OBJECT_ID = 13L; + + // data size in bytes (unreplicated) + private static final long FILE_ONE_SIZE = 500L; // bin 0 + private static final long FILE_TWO_SIZE = OzoneConsts.KB + 1; // bin 1 + private static final long FILE_THREE_SIZE = 4 * OzoneConsts.KB + 1; // bin 3 + private static final long FILE_FOUR_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 + private static final long FILE_FIVE_SIZE = 100L; // bin 0 + private static final long FILE_EIGHT_SIZE = OzoneConsts.KB + 1; // bin 1 + private static final long FILE_NINE_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 + private static final long FILE_TEN_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 + private static final long FILE_ELEVEN_SIZE = OzoneConsts.KB + 1; // bin 1 + + // replicated sizes (RATIS/THREE => 3x) + private static final long FILE1_REPLICA = repl(FILE_ONE_SIZE); + private static final long FILE2_REPLICA = repl(FILE_TWO_SIZE); + private static final long FILE3_REPLICA = repl(FILE_THREE_SIZE); + private static final long FILE4_REPLICA = repl(FILE_FOUR_SIZE); + private static final long FILE5_REPLICA = repl(FILE_FIVE_SIZE); + private static final long FILE8_REPLICA = repl(FILE_EIGHT_SIZE); + private static final long FILE9_REPLICA = repl(FILE_NINE_SIZE); + private static final long FILE10_REPLICA = repl(FILE_TEN_SIZE); + private static final long FILE11_REPLICA = repl(FILE_ELEVEN_SIZE); + + private static final long REPLICA_UNDER_ROOT = + FILE1_REPLICA + FILE2_REPLICA + FILE3_REPLICA + FILE4_REPLICA + + FILE5_REPLICA + FILE8_REPLICA + FILE9_REPLICA + FILE10_REPLICA + + FILE11_REPLICA; + private static final long REPLICA_UNDER_VOL = + FILE1_REPLICA + FILE2_REPLICA + FILE3_REPLICA + FILE4_REPLICA + + FILE5_REPLICA; + private static final long REPLICA_UNDER_BUCKET1 = + FILE1_REPLICA + FILE2_REPLICA + FILE3_REPLICA; + private static final long REPLICA_UNDER_BUCKET3 = + FILE8_REPLICA + FILE9_REPLICA + FILE10_REPLICA; + private static final long REPLICA_UNDER_KEY = FILE4_REPLICA; + + // quota in bytes + private static final long ROOT_QUOTA = 2 * (2 * OzoneConsts.MB); + private static final long VOL_QUOTA = 2 * OzoneConsts.MB; + private static final long VOL_TWO_QUOTA = 2 * OzoneConsts.MB; + private static final long BUCKET_ONE_QUOTA = OzoneConsts.MB; + private static final long BUCKET_TWO_QUOTA = OzoneConsts.MB; + private static final long BUCKET_THREE_QUOTA = OzoneConsts.MB; + private static final long BUCKET_FOUR_QUOTA = OzoneConsts.MB; + + // request paths + private static final String VOL_PATH = "/vol"; + private static final String VOL_TWO_PATH = "/vol2"; + private static final String BUCKET_ONE_PATH = "/vol/bucket1"; + private static final String BUCKET_TWO_PATH = "/vol/bucket2"; + private static final String BUCKET_THREE_PATH = "/vol2/bucket3"; + private static final String BUCKET_FOUR_PATH = "/vol2/bucket4"; + private static final String KEY_ONE_PATH = "/vol/bucket1/" + KEY_ONE; + private static final String KEY_TWO_PATH = "/vol/bucket1/" + KEY_TWO; + private static final String KEY_FIVE_PATH = "/vol/bucket2/" + KEY_FIVE; + private static final String KEY_EIGHT_PATH = "/vol2/bucket3/" + KEY_EIGHT; + private static final String KEY_ELEVEN_PATH = "/vol2/bucket4/" + KEY_ELEVEN; + private static final String MULTI_BLOCK_KEY_PATH = "/vol/bucket1/" + KEY_THREE; + + // unreplicated (namespace) data sizes + private static final long ROOT_DATA_SIZE = REPLICA_UNDER_ROOT; + private static final long VOL_DATA_SIZE = FILE_ONE_SIZE + FILE_TWO_SIZE + + FILE_THREE_SIZE + FILE_FOUR_SIZE + FILE_FIVE_SIZE; + private static final long VOL_TWO_DATA_SIZE = + FILE_EIGHT_SIZE + FILE_NINE_SIZE + FILE_TEN_SIZE + FILE_ELEVEN_SIZE; + private static final long BUCKET_ONE_DATA_SIZE = + FILE_ONE_SIZE + FILE_TWO_SIZE + FILE_THREE_SIZE; + private static final long BUCKET_TWO_DATA_SIZE = + FILE_FOUR_SIZE + FILE_FIVE_SIZE; + private static final long BUCKET_THREE_DATA_SIZE = + FILE_EIGHT_SIZE + FILE_NINE_SIZE + FILE_TEN_SIZE; + private static final long BUCKET_FOUR_DATA_SIZE = FILE_ELEVEN_SIZE; + + public FlatNSSummaryScenario() { + super("OBS_AND_LEGACY"); + } + + private static long repl(long size) { + return getReplicatedSize(size, RatisReplicationConfig.getInstance(THREE)); + } + + private static BucketLayout obs() { + return BucketLayout.OBJECT_STORE; + } + + private static BucketLayout legacy() { + return BucketLayout.LEGACY; + } + + @Override + boolean isFlatLayout() { + return true; + } + + @Override + long rootQuota() { + return ROOT_QUOTA; + } + + @Override + long rootDataSize() { + return ROOT_DATA_SIZE; + } + + @Override + OzoneConfiguration newConfiguration() { + OzoneConfiguration conf = new OzoneConfiguration(); + // Legacy buckets behave like OBS buckets with filesystem paths disabled. + conf.set(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, "false"); + return conf; + } + + @Override + OzoneManagerServiceProviderImpl mockOmServiceProvider() throws IOException { + return getMockOzoneManagerServiceProvider(); + } + + @Override + OMMetadataManager initializeOmMetadataManager(File omDbDir, + OzoneConfiguration conf) throws IOException { + conf.set(OZONE_OM_DB_DIRS, omDbDir.getAbsolutePath()); + conf.set(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, "false"); + // Make the Recon metadata manager (and thus the endpoint's bucket-handler + // selection) observe filesystem-paths=false, so Legacy buckets are treated + // as flat object stores. Required because getTestReconOmMetadataManager + // reads a shared static configuration that a prior scenario may have set. + setConfiguration(conf); + OMMetadataManager omMetadataManager = new OmMetadataManagerImpl(conf, null); + + putVolume(omMetadataManager, VOL, VOL_OBJECT_ID, VOL_QUOTA); + putVolume(omMetadataManager, VOL_TWO, VOL_TWO_OBJECT_ID, VOL_TWO_QUOTA); + putBucket(omMetadataManager, VOL, BUCKET_ONE, BUCKET_ONE_OBJECT_ID, + BUCKET_ONE_QUOTA, obs()); + putBucket(omMetadataManager, VOL, BUCKET_TWO, BUCKET_TWO_OBJECT_ID, + BUCKET_TWO_QUOTA, obs()); + putBucket(omMetadataManager, VOL_TWO, BUCKET_THREE, BUCKET_THREE_OBJECT_ID, + BUCKET_THREE_QUOTA, legacy()); + putBucket(omMetadataManager, VOL_TWO, BUCKET_FOUR, BUCKET_FOUR_OBJECT_ID, + BUCKET_FOUR_QUOTA, legacy()); + return omMetadataManager; + } + + private void putVolume(OMMetadataManager omMetadataManager, String volume, + long objectId, long quota) throws IOException { + omMetadataManager.getVolumeTable().put( + omMetadataManager.getVolumeKey(volume), + OmVolumeArgs.newBuilder() + .setObjectID(objectId) + .setVolume(volume) + .setAdminName(TEST_USER) + .setOwnerName(TEST_USER) + .setQuotaInBytes(quota) + .build()); + } + + private void putBucket(OMMetadataManager omMetadataManager, String volume, + String bucket, long objectId, long quota, BucketLayout layout) + throws IOException { + omMetadataManager.getBucketTable().put( + omMetadataManager.getBucketKey(volume, bucket), + OmBucketInfo.newBuilder() + .setVolumeName(volume) + .setBucketName(bucket) + .setObjectID(objectId) + .setQuotaInBytes(quota) + .setBucketLayout(layout) + .build()); + } + + @Override + void populateAndReprocess( + ReconNamespaceSummaryManager reconNamespaceSummaryManager, + ReconOMMetadataManager recon, OzoneConfiguration conf) throws Exception { + OmKeyLocationInfoGroup group1 = + NSSummaryEndpointTestBase.getLocationInfoGroup1(); + OmKeyLocationInfoGroup group2 = + NSSummaryEndpointTestBase.getLocationInfoGroup2(); + + writeReplicatedKey(recon, KEY_ONE, BUCKET_ONE, VOL, KEY_ONE_OBJECT_ID, + BUCKET_ONE_OBJECT_ID, VOL_OBJECT_ID, group1, FILE_ONE_SIZE, obs()); + writeReplicatedKey(recon, KEY_TWO, BUCKET_ONE, VOL, KEY_TWO_OBJECT_ID, + BUCKET_ONE_OBJECT_ID, VOL_OBJECT_ID, group2, FILE_TWO_SIZE, obs()); + writeReplicatedKey(recon, KEY_THREE, BUCKET_ONE, VOL, KEY_THREE_OBJECT_ID, + BUCKET_ONE_OBJECT_ID, VOL_OBJECT_ID, group1, FILE_THREE_SIZE, obs()); + writeReplicatedKey(recon, KEY_FOUR, BUCKET_TWO, VOL, KEY_FOUR_OBJECT_ID, + BUCKET_TWO_OBJECT_ID, VOL_OBJECT_ID, group2, FILE_FOUR_SIZE, obs()); + writeReplicatedKey(recon, KEY_FIVE, BUCKET_TWO, VOL, KEY_FIVE_OBJECT_ID, + BUCKET_TWO_OBJECT_ID, VOL_OBJECT_ID, group1, FILE_FIVE_SIZE, obs()); + writeReplicatedKey(recon, KEY_EIGHT, BUCKET_THREE, VOL_TWO, + KEY_EIGHT_OBJECT_ID, BUCKET_THREE_OBJECT_ID, VOL_TWO_OBJECT_ID, group2, + FILE_EIGHT_SIZE, legacy()); + writeReplicatedKey(recon, KEY_NINE, BUCKET_THREE, VOL_TWO, + KEY_NINE_OBJECT_ID, BUCKET_THREE_OBJECT_ID, VOL_TWO_OBJECT_ID, group1, + FILE_NINE_SIZE, legacy()); + writeReplicatedKey(recon, KEY_TEN, BUCKET_THREE, VOL_TWO, KEY_TEN_OBJECT_ID, + BUCKET_THREE_OBJECT_ID, VOL_TWO_OBJECT_ID, group2, FILE_TEN_SIZE, + legacy()); + writeReplicatedKey(recon, KEY_ELEVEN, BUCKET_FOUR, VOL_TWO, + KEY_ELEVEN_OBJECT_ID, BUCKET_FOUR_OBJECT_ID, VOL_TWO_OBJECT_ID, group1, + FILE_ELEVEN_SIZE, legacy()); + + new NSSummaryTaskWithOBS(reconNamespaceSummaryManager, recon, 10, 5, 20, + 2000).reprocessWithOBS(recon); + new NSSummaryTaskWithLegacy(reconNamespaceSummaryManager, recon, conf, 10) + .reprocessWithLegacy(recon); + } + + @SuppressWarnings("checkstyle:ParameterNumber") + private void writeReplicatedKey(ReconOMMetadataManager recon, String key, + String bucket, String vol, long keyObjectId, long bucketObjectId, + long volObjectId, OmKeyLocationInfoGroup locationGroup, long dataSize, + BucketLayout layout) throws IOException { + writeKeyToOm(recon, key, bucket, vol, key, keyObjectId, + bucketObjectId, bucketObjectId, volObjectId, + Collections.singletonList(locationGroup), layout, dataSize, + RatisReplicationConfig.getInstance(THREE)); + } + + @Override + void writeMultiBlockKey(ReconOMMetadataManager recon) throws IOException { + writeReplicatedKey(recon, MULTI_BLOCK_FILE, BUCKET_ONE, VOL, + MULTI_BLOCK_KEY_OBJECT_ID, BUCKET_ONE_OBJECT_ID, VOL_OBJECT_ID, + NSSummaryEndpointTestBase.getLocationInfoGroup1(), FILE_THREE_SIZE, + obs()); + } + + @Override + String multiBlockKeyPath() { + return MULTI_BLOCK_KEY_PATH; + } + + @Override + long multiBlockKeyReplicatedSize() { + return FILE3_REPLICA; + } + + // --------------------------------------------------------------------------- + // Assertions common to every layout + // --------------------------------------------------------------------------- + + @Override + void assertBasicInfoRoot(NSSummaryEndpoint endpoint, + ReconOMMetadataManager recon) throws Exception { + NamespaceSummaryResponse root = (NamespaceSummaryResponse) + endpoint.getBasicInfo(ROOT_PATH).getEntity(); + assertEquals(EntityType.ROOT, root.getEntityType()); + assertEquals(2, root.getCountStats().getNumVolume()); + assertEquals(4, root.getCountStats().getNumBucket()); + assertEquals(9, root.getCountStats().getNumTotalKey()); + } + + @Override + void assertBasicInfoVolume(NSSummaryEndpoint endpoint) throws Exception { + NamespaceSummaryResponse vol = (NamespaceSummaryResponse) + endpoint.getBasicInfo(VOL_PATH).getEntity(); + assertEquals(EntityType.VOLUME, vol.getEntityType()); + assertEquals(2, vol.getCountStats().getNumBucket()); + assertEquals(5, vol.getCountStats().getNumTotalKey()); + assertEquals(TEST_USER, + ((VolumeObjectDBInfo) vol.getObjectDBInfo()).getAdmin()); + assertEquals(TEST_USER, + ((VolumeObjectDBInfo) vol.getObjectDBInfo()).getOwner()); + assertEquals(VOL, vol.getObjectDBInfo().getName()); + assertEquals(2097152, vol.getObjectDBInfo().getQuotaInBytes()); + assertEquals(-1, vol.getObjectDBInfo().getQuotaInNamespace()); + } + + @Override + void assertBasicInfoBucketOne(NSSummaryEndpoint endpoint) throws Exception { + assertObsBucket(endpoint, BUCKET_ONE_PATH, BUCKET_ONE, VOL, 3); + } + + @Override + void assertBasicInfoBucketTwo(NSSummaryEndpoint endpoint) throws Exception { + assertObsBucket(endpoint, BUCKET_TWO_PATH, BUCKET_TWO, VOL, 2); + } + + void assertBasicInfoVolTwo(NSSummaryEndpoint endpoint) throws Exception { + NamespaceSummaryResponse vol = (NamespaceSummaryResponse) + endpoint.getBasicInfo(VOL_TWO_PATH).getEntity(); + assertEquals(EntityType.VOLUME, vol.getEntityType()); + assertEquals(2, vol.getCountStats().getNumBucket()); + assertEquals(4, vol.getCountStats().getNumTotalKey()); + assertEquals(TEST_USER, + ((VolumeObjectDBInfo) vol.getObjectDBInfo()).getAdmin()); + assertEquals(TEST_USER, + ((VolumeObjectDBInfo) vol.getObjectDBInfo()).getOwner()); + assertEquals(VOL_TWO, vol.getObjectDBInfo().getName()); + assertEquals(2097152, vol.getObjectDBInfo().getQuotaInBytes()); + assertEquals(-1, vol.getObjectDBInfo().getQuotaInNamespace()); + } + + void assertBasicInfoBucketThree(NSSummaryEndpoint endpoint) throws Exception { + assertLegacyBucket(endpoint, BUCKET_THREE_PATH, BUCKET_THREE, VOL_TWO, 3); + } + + void assertBasicInfoBucketFour(NSSummaryEndpoint endpoint) throws Exception { + assertLegacyBucket(endpoint, BUCKET_FOUR_PATH, BUCKET_FOUR, VOL_TWO, 1); + } + + private void assertObsBucket(NSSummaryEndpoint endpoint, String path, + String bucketName, String volName, int numKeys) throws Exception { + assertBucket(endpoint, path, bucketName, volName, numKeys, obs()); + } + + private void assertLegacyBucket(NSSummaryEndpoint endpoint, String path, + String bucketName, String volName, int numKeys) throws Exception { + assertBucket(endpoint, path, bucketName, volName, numKeys, legacy()); + } + + private void assertBucket(NSSummaryEndpoint endpoint, String path, + String bucketName, String volName, int numKeys, BucketLayout layout) + throws Exception { + NamespaceSummaryResponse bucket = (NamespaceSummaryResponse) + endpoint.getBasicInfo(path).getEntity(); + assertEquals(EntityType.BUCKET, bucket.getEntityType()); + assertEquals(numKeys, bucket.getCountStats().getNumTotalKey()); + assertEquals(volName, + ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getVolumeName()); + assertEquals(StorageType.DISK, + ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getStorageType()); + assertEquals(layout, + ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getBucketLayout()); + assertEquals(bucketName, + ((BucketObjectDBInfo) bucket.getObjectDBInfo()).getName()); + } + + @Override + void assertDiskUsageRoot(NSSummaryEndpoint endpoint) throws Exception { + DUResponse duRootRes = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, ROOT_PATH, false); + assertEquals(2, duRootRes.getCount()); + List duData = duRootRes.getDuData(); + Collections.sort(duData, + Comparator.comparing(DUResponse.DiskUsage::getSubpath)); + assertEquals(VOL_PATH, duData.get(0).getSubpath()); + assertEquals(VOL_TWO_PATH, duData.get(1).getSubpath()); + assertEquals(VOL_DATA_SIZE, duData.get(0).getSize()); + assertEquals(VOL_TWO_DATA_SIZE, duData.get(1).getSize()); + } + + @Override + void assertDiskUsageVolume(NSSummaryEndpoint endpoint) throws Exception { + DUResponse duVolRes = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, VOL_PATH, false); + assertEquals(2, duVolRes.getCount()); + List duData = duVolRes.getDuData(); + Collections.sort(duData, + Comparator.comparing(DUResponse.DiskUsage::getSubpath)); + assertEquals(BUCKET_ONE_PATH, duData.get(0).getSubpath()); + assertEquals(BUCKET_TWO_PATH, duData.get(1).getSubpath()); + assertEquals(BUCKET_ONE_DATA_SIZE, duData.get(0).getSize()); + assertEquals(BUCKET_TWO_DATA_SIZE, duData.get(1).getSize()); + } + + void assertDiskUsageVolTwo(NSSummaryEndpoint endpoint) throws Exception { + DUResponse duVolRes = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, VOL_TWO_PATH, false); + assertEquals(2, duVolRes.getCount()); + List duData = duVolRes.getDuData(); + Collections.sort(duData, + Comparator.comparing(DUResponse.DiskUsage::getSubpath)); + assertEquals(BUCKET_THREE_PATH, duData.get(0).getSubpath()); + assertEquals(BUCKET_FOUR_PATH, duData.get(1).getSubpath()); + assertEquals(VOL_TWO_DATA_SIZE, duVolRes.getSize()); + } + + /** OBS/Legacy buckets have no sub-paths unless files are explicitly listed. */ + void assertDiskUsageBucketOne(NSSummaryEndpoint endpoint) throws Exception { + assertFlatBucketDiskUsage(endpoint, BUCKET_ONE_PATH, 3, BUCKET_ONE_DATA_SIZE); + } + + void assertDiskUsageBucketTwo(NSSummaryEndpoint endpoint) throws Exception { + assertFlatBucketDiskUsage(endpoint, BUCKET_TWO_PATH, 2, BUCKET_TWO_DATA_SIZE); + } + + void assertDiskUsageBucketThree(NSSummaryEndpoint endpoint) throws Exception { + assertFlatBucketDiskUsage(endpoint, BUCKET_THREE_PATH, 3, + BUCKET_THREE_DATA_SIZE); + } + + private void assertFlatBucketDiskUsage(NSSummaryEndpoint endpoint, String path, + int numFiles, long dataSize) throws Exception { + DUResponse duBucketResponse = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, path, false); + assertEquals(0, duBucketResponse.getCount()); + DUResponse withFiles = (DUResponse) endpoint + .getDiskUsage(path, true, false, false).getEntity(); + assertEquals(numFiles, withFiles.getCount()); + assertEquals(dataSize, duBucketResponse.getSize()); + } + + @Override + void assertDiskUsageKey(NSSummaryEndpoint endpoint) throws Exception { + DUResponse duKeyResponse = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, KEY_PATH, false); + assertEquals(0, duKeyResponse.getCount()); + assertEquals(FILE_FOUR_SIZE, duKeyResponse.getSize()); + } + + void assertDiskUsageKeys(NSSummaryEndpoint endpoint) throws Exception { + assertKeySize(endpoint, KEY_ONE_PATH, FILE_ONE_SIZE); + assertKeySize(endpoint, KEY_TWO_PATH, FILE_TWO_SIZE); + assertKeySize(endpoint, KEY_FIVE_PATH, FILE_FIVE_SIZE); + assertKeySize(endpoint, KEY_EIGHT_PATH, FILE_EIGHT_SIZE); + assertKeySize(endpoint, KEY_ELEVEN_PATH, FILE_ELEVEN_SIZE); + } + + private void assertKeySize(NSSummaryEndpoint endpoint, String path, + long expectedSize) throws Exception { + DUResponse duKeyResponse = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, path, false); + assertEquals(0, duKeyResponse.getCount()); + assertEquals(expectedSize, duKeyResponse.getSize()); + } + + @Override + void assertQuotaUsage(NSSummaryEndpoint endpoint) throws Exception { + QuotaUsageResponse quRootRes = (QuotaUsageResponse) + endpoint.getQuotaUsage(ROOT_PATH).getEntity(); + assertEquals(ROOT_QUOTA, quRootRes.getQuota()); + assertEquals(ROOT_DATA_SIZE, quRootRes.getQuotaUsed()); + + assertQuota(endpoint, VOL_PATH, VOL_QUOTA, VOL_DATA_SIZE); + assertQuota(endpoint, BUCKET_ONE_PATH, BUCKET_ONE_QUOTA, + BUCKET_ONE_DATA_SIZE); + assertQuota(endpoint, BUCKET_TWO_PATH, BUCKET_TWO_QUOTA, + BUCKET_TWO_DATA_SIZE); + assertQuota(endpoint, BUCKET_THREE_PATH, BUCKET_THREE_QUOTA, + BUCKET_THREE_DATA_SIZE); + assertQuota(endpoint, BUCKET_FOUR_PATH, BUCKET_FOUR_QUOTA, + BUCKET_FOUR_DATA_SIZE); + + QuotaUsageResponse naKey = (QuotaUsageResponse) + endpoint.getQuotaUsage(KEY_PATH).getEntity(); + assertEquals(ResponseStatus.TYPE_NOT_APPLICABLE, naKey.getResponseCode()); + + QuotaUsageResponse invalid = (QuotaUsageResponse) + endpoint.getQuotaUsage(INVALID_PATH).getEntity(); + assertEquals(ResponseStatus.PATH_NOT_FOUND, invalid.getResponseCode()); + } + + private void assertQuota(NSSummaryEndpoint endpoint, String path, long quota, + long used) throws IOException { + QuotaUsageResponse res = + (QuotaUsageResponse) endpoint.getQuotaUsage(path).getEntity(); + assertEquals(quota, res.getQuota()); + assertEquals(used, res.getQuotaUsed()); + } + + @Override + void assertFileSizeDist(NSSummaryEndpoint endpoint) throws Exception { + NSSummaryEndpointTestBase.checkFileSizeDist(endpoint, ROOT_PATH, 2, 3, 3, 1); + NSSummaryEndpointTestBase.checkFileSizeDist(endpoint, VOL_PATH, 2, 1, 1, 1); + NSSummaryEndpointTestBase.checkFileSizeDist( + endpoint, BUCKET_ONE_PATH, 1, 1, 0, 1); + } + + @Override + void assertDataSizeUnderRootWithReplication(NSSummaryEndpoint endpoint) + throws IOException { + DUResponse res = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, ROOT_PATH, true); + assertEquals(ResponseStatus.OK, res.getStatus()); + assertEquals(REPLICA_UNDER_ROOT, res.getSizeWithReplica()); + assertEquals(REPLICA_UNDER_VOL, res.getDuData().get(0).getSizeWithReplica()); + } + + @Override + void assertDataSizeUnderVolWithReplication(NSSummaryEndpoint endpoint) + throws IOException { + DUResponse res = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, VOL_PATH, true); + assertEquals(ResponseStatus.OK, res.getStatus()); + assertEquals(REPLICA_UNDER_VOL, res.getSizeWithReplica()); + assertEquals(REPLICA_UNDER_BUCKET1, + res.getDuData().get(0).getSizeWithReplica()); + } + + void assertDataSizeUnderBucketOneWithReplication(NSSummaryEndpoint endpoint) + throws IOException { + DUResponse res = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, BUCKET_ONE_PATH, true); + assertEquals(ResponseStatus.OK, res.getStatus()); + assertEquals(REPLICA_UNDER_BUCKET1, res.getSizeWithReplica()); + } + + void assertDataSizeUnderBucketThreeWithReplication(NSSummaryEndpoint endpoint) + throws IOException { + DUResponse res = NSSummaryEndpointTestBase.getDiskUsage( + endpoint, BUCKET_THREE_PATH, true); + assertEquals(ResponseStatus.OK, res.getStatus()); + assertEquals(REPLICA_UNDER_BUCKET3, res.getSizeWithReplica()); + } + + @Override + void assertDataSizeUnderKeyWithReplication(NSSummaryEndpoint endpoint) + throws IOException { + DUResponse res = + NSSummaryEndpointTestBase.getDiskUsage(endpoint, KEY_PATH, true); + assertEquals(ResponseStatus.OK, res.getStatus()); + assertEquals(REPLICA_UNDER_KEY, res.getSizeWithReplica()); + } + + void assertNormalizePathUptoBucket() { + assertEquals("/", OmUtils.normalizePathUptoBucket(null)); + assertEquals("/", OmUtils.normalizePathUptoBucket("")); + assertEquals("volume1/bucket1/key1/key2", + OmUtils.normalizePathUptoBucket("///volume1/bucket1/key1/key2")); + assertEquals("volume1/bucket1", + OmUtils.normalizePathUptoBucket("volume1/bucket1")); + assertEquals("volume1/bucket1/key1/key2", + OmUtils.normalizePathUptoBucket("volume1/bucket1/key1/key2")); + assertEquals("volume1/bucket1/key1//key2", + OmUtils.normalizePathUptoBucket("volume1/bucket1/key1//key2")); + assertEquals("volume/bucket/key$%#1/./////////key$%#2", + OmUtils.normalizePathUptoBucket("volume/bucket/key$%#1/./////////key$%#2")); + } + + @Override + void verifyConstructFullPath(ReconOMMetadataManager recon, + ReconNamespaceSummaryManager reconNamespaceSummaryManager) + throws IOException { + assertFullPath(reconNamespaceSummaryManager, KEY_TWO, VOL, BUCKET_ONE, + KEY_TWO_OBJECT_ID, "vol/bucket1/" + KEY_TWO); + assertFullPath(reconNamespaceSummaryManager, KEY_FIVE, VOL, BUCKET_TWO, + KEY_FIVE_OBJECT_ID, "vol/bucket2/" + KEY_FIVE); + assertFullPath(reconNamespaceSummaryManager, KEY_EIGHT, VOL_TWO, + BUCKET_THREE, KEY_EIGHT_OBJECT_ID, "vol2/bucket3/" + KEY_EIGHT); + assertFullPath(reconNamespaceSummaryManager, KEY_ELEVEN, VOL_TWO, + BUCKET_FOUR, KEY_ELEVEN_OBJECT_ID, "vol2/bucket4/" + KEY_ELEVEN); + } + + private void assertFullPath(ReconNamespaceSummaryManager nsMgr, + String keyName, String vol, String bucket, long objectId, + String expected) throws IOException { + OmKeyInfo keyInfo = new OmKeyInfo.Builder() + .setKeyName(keyName) + .setVolumeName(vol) + .setBucketName(bucket) + .setObjectID(objectId) + .build(); + assertEquals(expected, ReconUtils.constructFullPath(keyInfo, nsMgr)); + } +} diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/FsoNSSummaryScenario.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/FsoNSSummaryScenario.java new file mode 100644 index 000000000000..ff93e6ed8beb --- /dev/null +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/FsoNSSummaryScenario.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.api; + +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getMockOzoneManagerServiceProviderWithFSO; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDirToOm; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.IOException; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; +import org.apache.hadoop.ozone.recon.ReconUtils; +import org.apache.hadoop.ozone.recon.api.types.NSSummary; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; +import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; +import org.apache.hadoop.ozone.recon.tasks.NSSummaryTaskWithFSO; + +/** + * FSO layout scenario. Directories live in the directory table and keys are + * parented by object ID. + */ +public class FsoNSSummaryScenario extends AbstractTreeNSSummaryScenario { + + public FsoNSSummaryScenario() { + super("FSO"); + } + + @Override + BucketLayout getBucketLayout() { + return BucketLayout.FILE_SYSTEM_OPTIMIZED; + } + + @Override + OzoneConfiguration newConfiguration() { + return new OzoneConfiguration(); + } + + @Override + void configureOmConfiguration(OzoneConfiguration conf) { + // FSO needs no extra OM configuration. + } + + @Override + OzoneManagerServiceProviderImpl mockOmServiceProvider() throws IOException { + return getMockOzoneManagerServiceProviderWithFSO(); + } + + @Override + void reprocess(ReconNamespaceSummaryManager reconNamespaceSummaryManager, + ReconOMMetadataManager reconOMMetadataManager, OzoneConfiguration conf) + throws IOException { + NSSummaryTaskWithFSO task = new NSSummaryTaskWithFSO( + reconNamespaceSummaryManager, reconOMMetadataManager, 10, 5, 20, 2000); + task.reprocessWithFSO(reconOMMetadataManager); + } + + @Override + @SuppressWarnings("checkstyle:ParameterNumber") + void writeDir(ReconOMMetadataManager recon, String dirName, String legacyPath, + long dirObjectId, long fsoParentObjectId, String bucketName, + String volName, long bucketObjectId, long volObjectId) + throws IOException { + writeDirToOm(recon, dirObjectId, fsoParentObjectId, bucketObjectId, + volObjectId, dirName); + } + + @Override + @SuppressWarnings("checkstyle:ParameterNumber") + void writeReplicatedKey(ReconOMMetadataManager recon, String key, + String bucket, String vol, String fileName, long keyObjectId, + long fsoParentObjectId, long bucketObjectId, long volObjectId, + OmKeyLocationInfoGroup locationGroup, long dataSize) throws IOException { + writeKeyToOm(recon, key, bucket, vol, fileName, keyObjectId, + fsoParentObjectId, bucketObjectId, volObjectId, + java.util.Collections.singletonList(locationGroup), getBucketLayout(), + dataSize, RatisReplicationConfig.getInstance(THREE)); + } + + @Override + void verifyConstructFullPath(ReconOMMetadataManager recon, + ReconNamespaceSummaryManager reconNamespaceSummaryManager) + throws IOException { + assertFullPath(reconNamespaceSummaryManager, FILE_TWO, VOL, BUCKET_ONE, + KEY_TWO_OBJECT_ID, DIR_TWO_OBJECT_ID, "vol/bucket1/dir1/dir2/file2"); + assertFullPath(reconNamespaceSummaryManager, FILE_THREE, VOL, BUCKET_ONE, + KEY_THREE_OBJECT_ID, DIR_THREE_OBJECT_ID, + "vol/bucket1/dir1/dir3/file3"); + assertFullPath(reconNamespaceSummaryManager, FILE_SIX, VOL, BUCKET_ONE, + KEY_SIX_OBJECT_ID, DIR_FOUR_OBJECT_ID, "vol/bucket1/dir1/dir4/file6"); + assertFullPath(reconNamespaceSummaryManager, FILE_ONE, VOL, BUCKET_ONE, + KEY_ONE_OBJECT_ID, BUCKET_ONE_OBJECT_ID, "vol/bucket1/file1"); + assertFullPath(reconNamespaceSummaryManager, FILE_NINE, VOL_TWO, + BUCKET_THREE, KEY_NINE_OBJECT_ID, DIR_FIVE_OBJECT_ID, + "vol2/bucket3/dir5/file9"); + + // When an NSSummary has parentId -1 (tree being rebuilt), constructFullPath + // returns an empty string. + NSSummary dir1Summary = + reconNamespaceSummaryManager.getNSSummary(DIR_ONE_OBJECT_ID); + dir1Summary.setParentId(-1); + reconNamespaceSummaryManager.deleteNSSummary(DIR_ONE_OBJECT_ID); + reconNamespaceSummaryManager.storeNSSummary(DIR_ONE_OBJECT_ID, dir1Summary); + assertEquals(-1, + reconNamespaceSummaryManager.getNSSummary(DIR_ONE_OBJECT_ID) + .getParentId(), + "The parentId should be updated to -1"); + + OmKeyInfo keyInfo = new OmKeyInfo.Builder() + .setKeyName(FILE_TWO) + .setVolumeName(VOL) + .setBucketName(BUCKET_ONE) + .setObjectID(KEY_TWO_OBJECT_ID) + .setParentObjectID(DIR_TWO_OBJECT_ID) + .build(); + assertEquals("", + ReconUtils.constructFullPath(keyInfo, reconNamespaceSummaryManager), + "Should return empty string when NSSummary tree is being rebuilt"); + } + + private void assertFullPath(ReconNamespaceSummaryManager nsMgr, + String keyName, String vol, String bucket, long objectId, + long parentObjectId, String expected) throws IOException { + OmKeyInfo keyInfo = new OmKeyInfo.Builder() + .setKeyName(keyName) + .setVolumeName(vol) + .setBucketName(bucket) + .setObjectID(objectId) + .setParentObjectID(parentObjectId) + .build(); + assertEquals(expected, ReconUtils.constructFullPath(keyInfo, nsMgr)); + } +} diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/LegacyNSSummaryScenario.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/LegacyNSSummaryScenario.java new file mode 100644 index 000000000000..a38254e336c3 --- /dev/null +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/LegacyNSSummaryScenario.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.api; + +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getMockOzoneManagerServiceProvider; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.setConfiguration; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDirToOm; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.IOException; +import java.util.Collections; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; +import org.apache.hadoop.ozone.recon.ReconUtils; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; +import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; +import org.apache.hadoop.ozone.recon.tasks.NSSummaryTaskWithLegacy; + +/** + * Legacy layout scenario with {@code ozone.om.enable.filesystem.paths=true}, so + * the namespace tree matches FSO. Directories and keys are stored in the key + * table keyed by their full path; the object-ID parent is not used. + */ +public class LegacyNSSummaryScenario extends AbstractTreeNSSummaryScenario { + + private static final long PARENT_OBJECT_ID_ZERO = 0L; + + public LegacyNSSummaryScenario() { + super("LEGACY"); + } + + @Override + BucketLayout getBucketLayout() { + return BucketLayout.LEGACY; + } + + @Override + OzoneConfiguration newConfiguration() { + return new OzoneConfiguration(); + } + + @Override + void configureOmConfiguration(OzoneConfiguration conf) { + conf.set(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, "true"); + setConfiguration(conf); + } + + @Override + OzoneManagerServiceProviderImpl mockOmServiceProvider() throws IOException { + return getMockOzoneManagerServiceProvider(); + } + + @Override + void reprocess(ReconNamespaceSummaryManager reconNamespaceSummaryManager, + ReconOMMetadataManager reconOMMetadataManager, OzoneConfiguration conf) + throws IOException { + NSSummaryTaskWithLegacy task = new NSSummaryTaskWithLegacy( + reconNamespaceSummaryManager, reconOMMetadataManager, conf, 10); + task.reprocessWithLegacy(reconOMMetadataManager); + } + + @Override + @SuppressWarnings("checkstyle:ParameterNumber") + void writeDir(ReconOMMetadataManager recon, String dirName, String legacyPath, + long dirObjectId, long fsoParentObjectId, String bucketName, + String volName, long bucketObjectId, long volObjectId) + throws IOException { + writeDirToOm(recon, legacyPath, bucketName, volName, dirName, dirObjectId, + PARENT_OBJECT_ID_ZERO, bucketObjectId, volObjectId, getBucketLayout()); + } + + @Override + @SuppressWarnings("checkstyle:ParameterNumber") + void writeReplicatedKey(ReconOMMetadataManager recon, String key, + String bucket, String vol, String fileName, long keyObjectId, + long fsoParentObjectId, long bucketObjectId, long volObjectId, + OmKeyLocationInfoGroup locationGroup, long dataSize) throws IOException { + writeKeyToOm(recon, key, bucket, vol, fileName, keyObjectId, + PARENT_OBJECT_ID_ZERO, bucketObjectId, volObjectId, + Collections.singletonList(locationGroup), getBucketLayout(), dataSize, + RatisReplicationConfig.getInstance(THREE)); + } + + @Override + void verifyConstructFullPath(ReconOMMetadataManager recon, + ReconNamespaceSummaryManager reconNamespaceSummaryManager) + throws IOException { + // For key tables the parent object ID is not set; it defaults to -1 in the + // NSSummary, so full paths are reconstructed from the key name. + assertFullPath(reconNamespaceSummaryManager, "dir1/dir2/file2", + KEY_TWO_OBJECT_ID, "vol/bucket1/dir1/dir2/file2"); + assertFullPath(reconNamespaceSummaryManager, "dir1/dir2/", + DIR_TWO_OBJECT_ID, "vol/bucket1/dir1/dir2/"); + assertFullPath(reconNamespaceSummaryManager, "dir1/dir4/file6", + KEY_SIX_OBJECT_ID, "vol/bucket1/dir1/dir4/file6"); + } + + private void assertFullPath(ReconNamespaceSummaryManager nsMgr, + String keyName, long objectId, String expected) throws IOException { + OmKeyInfo keyInfo = new OmKeyInfo.Builder() + .setKeyName(keyName) + .setVolumeName(VOL) + .setBucketName(BUCKET_ONE) + .setObjectID(objectId) + .build(); + assertEquals(expected, ReconUtils.constructFullPath(keyInfo, nsMgr)); + } +} diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpointTestBase.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpointTestBase.java new file mode 100644 index 000000000000..18a39ab85b06 --- /dev/null +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/NSSummaryEndpointTestBase.java @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.api; + +import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import javax.ws.rs.core.Response; +import org.apache.hadoop.hdds.client.BlockID; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State; +import org.apache.hadoop.hdds.scm.container.ContainerID; +import org.apache.hadoop.hdds.scm.container.ContainerManager; +import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; +import org.apache.hadoop.hdds.scm.container.ContainerReplica; +import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; +import org.apache.hadoop.ozone.recon.ReconConstants; +import org.apache.hadoop.ozone.recon.api.types.DUResponse; +import org.apache.hadoop.ozone.recon.api.types.EntityType; +import org.apache.hadoop.ozone.recon.api.types.FileSizeDistributionResponse; +import org.apache.hadoop.ozone.recon.api.types.KeyObjectDBInfo; +import org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse; +import org.apache.hadoop.ozone.recon.api.types.ResponseStatus; +import org.apache.hadoop.ozone.recon.scm.ReconNodeManager; +import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade; + +/** + * Layout-independent scaffolding shared by the parameterized + * {@link TestNSSummaryEndpoint} suite. + * + *

Holds the mock SCM wiring, the block/container fixtures used to build + * multi-block keys, and the assertion helpers that are identical for every + * bucket layout. Everything that varies per layout (namespace tree, object + * IDs, expected sizes) lives in {@link NSSummaryTestScenario}. + */ +public abstract class NSSummaryEndpointTestBase { + + // container IDs backing the multi-block keys + static final long CONTAINER_ONE_ID = 1L; + static final long CONTAINER_TWO_ID = 2L; + static final long CONTAINER_THREE_ID = 3L; + static final long CONTAINER_FOUR_ID = 4L; + static final long CONTAINER_FIVE_ID = 5L; + static final long CONTAINER_SIX_ID = 6L; + + // container replica counts used only by the Container/Node/ClusterState + // endpoints. The NSSummary DU path derives sizeWithReplica from each key's + // own ReplicationConfig (OmKeyInfo#getReplicatedSize), so these counts do + // not influence any assertion in this suite; they exist purely so the mock + // ReconSCM binding is complete. + private static final int CONTAINER_ONE_REPLICA_COUNT = 3; + private static final int CONTAINER_TWO_REPLICA_COUNT = 2; + private static final int CONTAINER_THREE_REPLICA_COUNT = 4; + private static final int CONTAINER_FOUR_REPLICA_COUNT = 5; + private static final int CONTAINER_FIVE_REPLICA_COUNT = 2; + private static final int CONTAINER_SIX_REPLICA_COUNT = 3; + + // block lengths + private static final long BLOCK_ONE_LENGTH = 1000L; + private static final long BLOCK_TWO_LENGTH = 2000L; + private static final long BLOCK_THREE_LENGTH = 3000L; + private static final long BLOCK_FOUR_LENGTH = 4000L; + private static final long BLOCK_FIVE_LENGTH = 5000L; + private static final long BLOCK_SIX_LENGTH = 6000L; + + /** + * Location group over containers 1, 2 and 3. + */ + static OmKeyLocationInfoGroup getLocationInfoGroup1() { + List locationInfoList = new ArrayList<>(); + locationInfoList.add(new OmKeyLocationInfo.Builder() + .setBlockID(new BlockID(CONTAINER_ONE_ID, 0L)) + .setLength(BLOCK_ONE_LENGTH) + .build()); + locationInfoList.add(new OmKeyLocationInfo.Builder() + .setBlockID(new BlockID(CONTAINER_TWO_ID, 0L)) + .setLength(BLOCK_TWO_LENGTH) + .build()); + locationInfoList.add(new OmKeyLocationInfo.Builder() + .setBlockID(new BlockID(CONTAINER_THREE_ID, 0L)) + .setLength(BLOCK_THREE_LENGTH) + .build()); + return new OmKeyLocationInfoGroup(0L, locationInfoList); + } + + /** + * Location group over containers 4, 5 and 6. + */ + static OmKeyLocationInfoGroup getLocationInfoGroup2() { + List locationInfoList = new ArrayList<>(); + locationInfoList.add(new OmKeyLocationInfo.Builder() + .setBlockID(new BlockID(CONTAINER_FOUR_ID, 0L)) + .setLength(BLOCK_FOUR_LENGTH) + .build()); + locationInfoList.add(new OmKeyLocationInfo.Builder() + .setBlockID(new BlockID(CONTAINER_FIVE_ID, 0L)) + .setLength(BLOCK_FIVE_LENGTH) + .build()); + locationInfoList.add(new OmKeyLocationInfo.Builder() + .setBlockID(new BlockID(CONTAINER_SIX_ID, 0L)) + .setLength(BLOCK_SIX_LENGTH) + .build()); + return new OmKeyLocationInfoGroup(0L, locationInfoList); + } + + /** + * Generate a set of mock container replicas for a container. + * @param replicationFactor number of replicas + * @param containerID the container being replicated + * @return a set of container replicas for testing + */ + private static Set generateMockContainerReplicas( + int replicationFactor, ContainerID containerID) { + Set result = new HashSet<>(); + for (int i = 0; i < replicationFactor; ++i) { + DatanodeDetails randomDatanode = randomDatanodeDetails(); + result.add(new ContainerReplica.ContainerReplicaBuilder() + .setContainerID(containerID) + .setContainerState(State.OPEN) + .setDatanodeDetails(randomDatanode) + .build()); + } + return result; + } + + static ReconStorageContainerManagerFacade getMockReconSCM( + long rootQuota, long rootDataSize) throws ContainerNotFoundException { + ReconStorageContainerManagerFacade reconSCM = + mock(ReconStorageContainerManagerFacade.class); + ContainerManager containerManager = mock(ContainerManager.class); + + int[] replicaCounts = { + CONTAINER_ONE_REPLICA_COUNT, CONTAINER_TWO_REPLICA_COUNT, + CONTAINER_THREE_REPLICA_COUNT, CONTAINER_FOUR_REPLICA_COUNT, + CONTAINER_FIVE_REPLICA_COUNT, CONTAINER_SIX_REPLICA_COUNT}; + for (int i = 0; i < replicaCounts.length; i++) { + ContainerID containerID = ContainerID.valueOf(i + 1L); + when(containerManager.getContainerReplicas(containerID)).thenReturn( + generateMockContainerReplicas(replicaCounts[i], containerID)); + } + + when(reconSCM.getContainerManager()).thenReturn(containerManager); + ReconNodeManager mockReconNodeManager = mock(ReconNodeManager.class); + when(mockReconNodeManager.getStats()) + .thenReturn(getMockSCMRootStat(rootQuota, rootDataSize)); + when(reconSCM.getScmNodeManager()).thenReturn(mockReconNodeManager); + return reconSCM; + } + + private static SCMNodeStat getMockSCMRootStat(long rootQuota, + long rootDataSize) { + return new SCMNodeStat(rootQuota, rootDataSize, + rootQuota - rootDataSize, 0L, 0L, 0); + } + + static DUResponse getDiskUsage(NSSummaryEndpoint endpoint, String path, + boolean withReplica) throws IOException { + Response response = endpoint.getDiskUsage(path, false, withReplica, false); + return (DUResponse) response.getEntity(); + } + + static void checkFileSizeDist(NSSummaryEndpoint endpoint, String path, + int bin0, int bin1, int bin2, int bin3) throws Exception { + Response res = endpoint.getFileSizeDistribution(path); + FileSizeDistributionResponse fileSizeDistResObj = + (FileSizeDistributionResponse) res.getEntity(); + int[] fileSizeDist = fileSizeDistResObj.getFileSizeDist(); + assertEquals(bin0, fileSizeDist[0]); + assertEquals(bin1, fileSizeDist[1]); + assertEquals(bin2, fileSizeDist[2]); + assertEquals(bin3, fileSizeDist[3]); + for (int i = 4; i < ReconConstants.NUM_OF_FILE_SIZE_BINS; ++i) { + assertEquals(0, fileSizeDist[i]); + } + } + + static void assertBasicInfoNoPath(NSSummaryEndpoint endpoint, + String invalidPath) throws Exception { + Response invalidResponse = endpoint.getBasicInfo(invalidPath); + NamespaceSummaryResponse invalidObj = + (NamespaceSummaryResponse) invalidResponse.getEntity(); + assertEquals(ResponseStatus.PATH_NOT_FOUND, invalidObj.getStatus()); + assertNull(invalidObj.getCountStats()); + assertNull(invalidObj.getObjectDBInfo()); + } + + /** + * The key at {@code keyPath} (/vol/bucket2/file4) is written as a genuinely + * replicated (RATIS/THREE) multi-block key by every scenario, so the basic + * info must report the unreplicated data size and a RATIS replication type. + */ + static void assertBasicInfoKey(NSSummaryEndpoint endpoint, String keyPath, + long dataSize) throws Exception { + Response keyResponse = endpoint.getBasicInfo(keyPath); + NamespaceSummaryResponse keyResObj = + (NamespaceSummaryResponse) keyResponse.getEntity(); + assertEquals(EntityType.KEY, keyResObj.getEntityType()); + assertEquals("vol", + ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getVolumeName()); + assertEquals("bucket2", + ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getBucketName()); + assertEquals("file4", + ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getKeyName()); + assertEquals(dataSize, + ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getDataSize()); + assertEquals(HddsProtos.ReplicationType.RATIS, + ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()) + .getReplicationConfig().getReplicationType()); + } +} diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/NSSummaryTestScenario.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/NSSummaryTestScenario.java new file mode 100644 index 000000000000..eafb35a27097 --- /dev/null +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/NSSummaryTestScenario.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.api; + +import java.io.File; +import java.io.IOException; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; +import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; + +/** + * A single parameterization of {@link TestNSSummaryEndpoint}: it owns the + * layout-specific fixture (config, OM provider mock, namespace population and + * reprocess) and the expected-value assertions for that layout. + * + *

Two concrete shapes exist: a tree fixture (FSO and Legacy with + * filesystem-paths enabled, see {@link AbstractTreeNSSummaryScenario}) and a + * flat fixture (OBS + Legacy with filesystem-paths disabled, see + * {@link FlatNSSummaryScenario}). Behaviour that only one shape supports is + * gated in the suite via the capability flags below and reached through + * {@code (AbstractTreeNSSummaryScenario) scenario} / + * {@code (FlatNSSummaryScenario) scenario} casts. + */ +public abstract class NSSummaryTestScenario { + + static final String TEST_USER = "TestUser"; + + // request paths shared by every layout + static final String ROOT_PATH = "/"; + static final String INVALID_PATH = "/vol/path/not/found"; + static final String KEY_PATH = "/vol/bucket2/file4"; + + // path-parsing fixture, identical across layouts + static final String TEST_PATH_UTILITY = "/vol1/buck1/a/b/c/d/e/file1.txt"; + static final String PARENT_DIR = "vol1/buck1/a/b/c/d/e"; + static final String[] TEST_NAMES = + new String[]{"vol1", "buck1", "a", "b", "c", "d", "e", "file1.txt"}; + static final String TEST_KEY_NAMES = "a/b/c/d/e/file1.txt"; + + private final String displayName; + + protected NSSummaryTestScenario(String displayName) { + this.displayName = displayName; + } + + // --------------------------------------------------------------------------- + // Fixture construction hooks + // --------------------------------------------------------------------------- + + abstract OzoneConfiguration newConfiguration(); + + abstract OzoneManagerServiceProviderImpl mockOmServiceProvider() + throws IOException; + + abstract OMMetadataManager initializeOmMetadataManager(File omDbDir, + OzoneConfiguration conf) throws IOException; + + /** + * Populate the Recon OM DB with the full namespace (directories and + * genuinely-replicated keys) and reprocess it into the NSSummary RocksDB. + */ + abstract void populateAndReprocess( + ReconNamespaceSummaryManager reconNamespaceSummaryManager, + ReconOMMetadataManager reconOMMetadataManager, + OzoneConfiguration conf) throws Exception; + + /** + * Write the extra single multi-block key exercised by + * {@code testDiskUsageWithReplication}. + */ + abstract void writeMultiBlockKey( + ReconOMMetadataManager reconOMMetadataManager) throws IOException; + + abstract void verifyConstructFullPath( + ReconOMMetadataManager reconOMMetadataManager, + ReconNamespaceSummaryManager reconNamespaceSummaryManager) + throws IOException; + + // Values needed to build the mock SCM before the fixture exists. + abstract long rootQuota(); + + abstract long rootDataSize(); + + // --------------------------------------------------------------------------- + // Capability flags used by the suite to gate structure-specific tests + // --------------------------------------------------------------------------- + + boolean hasDirectories() { + return false; + } + + boolean hasVolumeThree() { + return false; + } + + boolean isFlatLayout() { + return false; + } + + // --------------------------------------------------------------------------- + // Assertions common to every layout (expectations differ per scenario) + // --------------------------------------------------------------------------- + + abstract void assertBasicInfoRoot(NSSummaryEndpoint endpoint, + ReconOMMetadataManager reconOMMetadataManager) throws Exception; + + abstract void assertBasicInfoVolume(NSSummaryEndpoint endpoint) + throws Exception; + + abstract void assertBasicInfoBucketOne(NSSummaryEndpoint endpoint) + throws Exception; + + abstract void assertBasicInfoBucketTwo(NSSummaryEndpoint endpoint) + throws Exception; + + abstract void assertDiskUsageRoot(NSSummaryEndpoint endpoint) + throws Exception; + + abstract void assertDiskUsageVolume(NSSummaryEndpoint endpoint) + throws Exception; + + abstract void assertDiskUsageKey(NSSummaryEndpoint endpoint) throws Exception; + + abstract void assertQuotaUsage(NSSummaryEndpoint endpoint) throws Exception; + + abstract void assertFileSizeDist(NSSummaryEndpoint endpoint) throws Exception; + + /** Expected replicated size of the {@link #writeMultiBlockKey} key. */ + abstract long multiBlockKeyReplicatedSize(); + + abstract String multiBlockKeyPath(); + + abstract void assertDataSizeUnderRootWithReplication( + NSSummaryEndpoint endpoint) throws IOException; + + abstract void assertDataSizeUnderVolWithReplication( + NSSummaryEndpoint endpoint) throws IOException; + + abstract void assertDataSizeUnderKeyWithReplication( + NSSummaryEndpoint endpoint) throws IOException; + + @Override + public String toString() { + return displayName; + } +} diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/NSSummaryTests.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/NSSummaryTests.java deleted file mode 100644 index 9f5ff85edfce..000000000000 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/NSSummaryTests.java +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.recon.api; - -import static java.util.Collections.singletonList; -import static java.util.Collections.singletonMap; -import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS; -import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType.USER; -import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.WRITE; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; - -import javax.ws.rs.core.Response; -import org.apache.hadoop.hdds.protocol.StorageType; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.ozone.OzoneAcl; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo; -import org.apache.hadoop.ozone.recon.api.types.BucketObjectDBInfo; -import org.apache.hadoop.ozone.recon.api.types.EntityType; -import org.apache.hadoop.ozone.recon.api.types.KeyObjectDBInfo; -import org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse; -import org.apache.hadoop.ozone.recon.api.types.ResponseStatus; -import org.apache.hadoop.ozone.recon.api.types.VolumeObjectDBInfo; -import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; - -/** - * Shared NSSummary test cases. - */ -public abstract class NSSummaryTests { - private static final String ROOT_PATH = "/"; - private static final String VOL_PATH = "/vol"; - private static final String BUCKET_ONE_PATH = "/vol/bucket1"; - private static final String BUCKET_TWO_PATH = "/vol/bucket2"; - private static final String DIR_ONE_PATH = "/vol/bucket1/dir1"; - private static final String INVALID_PATH = "/vol/path/not/found"; - private static final String KEY_PATH = "/vol/bucket2/file4"; - - public static void testNSSummaryBasicInfoRoot( - NSSummaryEndpoint nsSummaryEndpoint, - ReconOMMetadataManager reconOMMetadataManager) throws Exception { - String username = "myuser"; - OmPrefixInfo omPrefixInfo = OmPrefixInfo.newBuilder() - .setName(ROOT_PATH) - .setObjectID(10) - .setUpdateID(100) - .setAcls(singletonList(OzoneAcl.of(USER, username, ACCESS, WRITE))) - .addAllMetadata(singletonMap("key", "value")) - .build(); - reconOMMetadataManager.getPrefixTable() - .put(OzoneConsts.OM_KEY_PREFIX, omPrefixInfo); - // Test root basics - Response rootResponse = nsSummaryEndpoint.getBasicInfo(ROOT_PATH); - NamespaceSummaryResponse rootResponseObj = - (NamespaceSummaryResponse) rootResponse.getEntity(); - assertEquals(EntityType.ROOT, rootResponseObj.getEntityType()); - assertEquals(3, rootResponseObj.getCountStats().getNumVolume()); - assertEquals(5, rootResponseObj.getCountStats().getNumBucket()); - assertEquals(7, rootResponseObj.getCountStats().getNumTotalDir()); - assertEquals(14, rootResponseObj.getCountStats().getNumTotalKey()); - assertEquals("USER", - rootResponseObj.getObjectDBInfo().getAcls().get(0).getType()); - assertEquals("WRITE", rootResponseObj.getObjectDBInfo().getAcls().get(0) - .getAclList().get(0)); - assertEquals(username, - rootResponseObj.getObjectDBInfo().getAcls().get(0).getName()); - assertEquals("value", - rootResponseObj.getObjectDBInfo().getMetadata().get("key")); - assertEquals("ACCESS", - rootResponseObj.getObjectDBInfo().getAcls().get(0).getScope()); - } - - public void testNSSummaryBasicInfoVolume( - NSSummaryEndpoint nsSummaryEndpoint) throws Exception { - Response volResponse = nsSummaryEndpoint.getBasicInfo(VOL_PATH); - NamespaceSummaryResponse volResponseObj = - (NamespaceSummaryResponse) volResponse.getEntity(); - assertEquals(EntityType.VOLUME, - volResponseObj.getEntityType()); - assertEquals(2, volResponseObj.getCountStats().getNumBucket()); - assertEquals(4, volResponseObj.getCountStats().getNumTotalDir()); - assertEquals(7, volResponseObj.getCountStats().getNumTotalKey()); - assertEquals("TestUser", ((VolumeObjectDBInfo) volResponseObj. - getObjectDBInfo()).getAdmin()); - assertEquals("TestUser", ((VolumeObjectDBInfo) volResponseObj. - getObjectDBInfo()).getOwner()); - assertEquals("vol", volResponseObj.getObjectDBInfo().getName()); - assertEquals(2097152, volResponseObj.getObjectDBInfo().getQuotaInBytes()); - assertEquals(-1, volResponseObj.getObjectDBInfo().getQuotaInNamespace()); - } - - public void testNSSummaryBasicInfoBucketOne(BucketLayout bucketLayout, - NSSummaryEndpoint nsSummaryEndpoint) throws Exception { - Response bucketOneResponse = - nsSummaryEndpoint.getBasicInfo(BUCKET_ONE_PATH); - NamespaceSummaryResponse bucketOneObj = - (NamespaceSummaryResponse) bucketOneResponse.getEntity(); - assertEquals(EntityType.BUCKET, bucketOneObj.getEntityType()); - assertEquals(4, bucketOneObj.getCountStats().getNumTotalDir()); - assertEquals(5, bucketOneObj.getCountStats().getNumTotalKey()); - assertEquals("vol", - ((BucketObjectDBInfo) bucketOneObj.getObjectDBInfo()).getVolumeName()); - assertEquals(StorageType.DISK, - ((BucketObjectDBInfo) - bucketOneObj.getObjectDBInfo()).getStorageType()); - assertEquals(bucketLayout, - ((BucketObjectDBInfo) - bucketOneObj.getObjectDBInfo()).getBucketLayout()); - assertEquals("bucket1", - ((BucketObjectDBInfo) bucketOneObj.getObjectDBInfo()).getName()); - } - - public void testNSSummaryBasicInfoBucketTwo( - BucketLayout bucketLayout, - NSSummaryEndpoint nsSummaryEndpoint) throws Exception { - Response bucketTwoResponse = - nsSummaryEndpoint.getBasicInfo(BUCKET_TWO_PATH); - NamespaceSummaryResponse bucketTwoObj = - (NamespaceSummaryResponse) bucketTwoResponse.getEntity(); - assertEquals(EntityType.BUCKET, bucketTwoObj.getEntityType()); - assertEquals(0, bucketTwoObj.getCountStats().getNumTotalDir()); - assertEquals(2, bucketTwoObj.getCountStats().getNumTotalKey()); - assertEquals("vol", - ((BucketObjectDBInfo) bucketTwoObj.getObjectDBInfo()).getVolumeName()); - assertEquals(StorageType.DISK, - ((BucketObjectDBInfo) - bucketTwoObj.getObjectDBInfo()).getStorageType()); - assertEquals(bucketLayout, - ((BucketObjectDBInfo) - bucketTwoObj.getObjectDBInfo()).getBucketLayout()); - assertEquals("bucket2", - ((BucketObjectDBInfo) bucketTwoObj.getObjectDBInfo()).getName()); - } - - public void testNSSummaryBasicInfoDir( - NSSummaryEndpoint nsSummaryEndpoint) throws Exception { - Response dirOneResponse = nsSummaryEndpoint.getBasicInfo(DIR_ONE_PATH); - NamespaceSummaryResponse dirOneObj = - (NamespaceSummaryResponse) dirOneResponse.getEntity(); - assertEquals(EntityType.DIRECTORY, dirOneObj.getEntityType()); - assertEquals(3, dirOneObj.getCountStats().getNumTotalDir()); - assertEquals(4, dirOneObj.getCountStats().getNumTotalKey()); - assertEquals("dir1", dirOneObj.getObjectDBInfo().getName()); - assertEquals(0, dirOneObj.getObjectDBInfo().getMetadata().size()); - assertEquals(0, dirOneObj.getObjectDBInfo().getQuotaInBytes()); - assertEquals(0, dirOneObj.getObjectDBInfo().getQuotaInNamespace()); - assertEquals(0, dirOneObj.getObjectDBInfo().getUsedNamespace()); - } - - public void testNSSummaryBasicInfoNoPath( - NSSummaryEndpoint nsSummaryEndpoint) throws Exception { - Response invalidResponse = nsSummaryEndpoint - .getBasicInfo(INVALID_PATH); - NamespaceSummaryResponse invalidObj = - (NamespaceSummaryResponse) invalidResponse.getEntity(); - assertEquals(ResponseStatus.PATH_NOT_FOUND, invalidObj.getStatus()); - assertNull(invalidObj.getCountStats()); - assertNull(invalidObj.getObjectDBInfo()); - } - - public void testNSSummaryBasicInfoKey( - NSSummaryEndpoint nsSummaryEndpoint) throws Exception { - Response keyResponse = nsSummaryEndpoint.getBasicInfo(KEY_PATH); - NamespaceSummaryResponse keyResObj = - (NamespaceSummaryResponse) keyResponse.getEntity(); - assertEquals(EntityType.KEY, keyResObj.getEntityType()); - assertEquals("vol", - ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getVolumeName()); - assertEquals("bucket2", - ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getBucketName()); - assertEquals("file4", - ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getKeyName()); - assertEquals(2049, - ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()).getDataSize()); - assertEquals(HddsProtos.ReplicationType.STAND_ALONE, - ((KeyObjectDBInfo) keyResObj.getObjectDBInfo()). - getReplicationConfig().getReplicationType()); - } -} diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpoint.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpoint.java new file mode 100644 index 000000000000..e9e411fccf89 --- /dev/null +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpoint.java @@ -0,0 +1,389 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.api; + +import static org.apache.hadoop.ozone.OzoneConsts.KB; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; +import javax.ws.rs.core.Response; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.recon.ReconTestInjector; +import org.apache.hadoop.ozone.recon.ReconUtils; +import org.apache.hadoop.ozone.recon.api.handlers.BucketHandler; +import org.apache.hadoop.ozone.recon.api.handlers.EntityHandler; +import org.apache.hadoop.ozone.recon.api.types.DUResponse; +import org.apache.hadoop.ozone.recon.api.types.NSSummary; +import org.apache.hadoop.ozone.recon.api.types.ResponseStatus; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; +import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; +import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; +import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.Parameter; +import org.junit.jupiter.params.ParameterizedClass; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Tests for the NSSummary REST APIs, run against every bucket layout. + * + *

Each {@link NSSummaryTestScenario} builds its own namespace and owns the + * expected values, so the tree (FSO, Legacy) and flat (OBS + Legacy) fixtures + * share this single suite. Structure-specific tests are gated with + * {@link org.junit.jupiter.api.Assumptions} on the scenario capability flags. + */ +@ParameterizedClass +@MethodSource("scenarios") +public class TestNSSummaryEndpoint extends NSSummaryEndpointTestBase { + + @TempDir + private Path temporaryFolder; + + @Parameter + private NSSummaryTestScenario scenario; + + private ReconOMMetadataManager reconOMMetadataManager; + private ReconNamespaceSummaryManager reconNamespaceSummaryManager; + private NSSummaryEndpoint nsSummaryEndpoint; + + static Stream scenarios() { + return Stream.of( + new FsoNSSummaryScenario(), + new LegacyNSSummaryScenario(), + new FlatNSSummaryScenario()); + } + + @BeforeEach + public void setUp() throws Exception { + OzoneConfiguration conf = scenario.newConfiguration(); + OMMetadataManager omMetadataManager = scenario.initializeOmMetadataManager( + Files.createDirectory(temporaryFolder.resolve("JunitOmDBDir")).toFile(), + conf); + OzoneManagerServiceProviderImpl ozoneManagerServiceProvider = + scenario.mockOmServiceProvider(); + reconOMMetadataManager = getTestReconOmMetadataManager(omMetadataManager, + Files.createDirectory(temporaryFolder.resolve("omMetadataDir")) + .toFile()); + + ReconTestInjector reconTestInjector = + new ReconTestInjector.Builder(temporaryFolder.toFile()) + .withReconOm(reconOMMetadataManager) + .withOmServiceProvider(ozoneManagerServiceProvider) + .withReconSqlDb() + .withContainerDB() + .addBinding(OzoneStorageContainerManager.class, + getMockReconSCM(scenario.rootQuota(), scenario.rootDataSize())) + .addBinding(StorageContainerServiceProvider.class, + mock(StorageContainerServiceProviderImpl.class)) + .addBinding(NSSummaryEndpoint.class) + .build(); + reconNamespaceSummaryManager = + reconTestInjector.getInstance(ReconNamespaceSummaryManager.class); + nsSummaryEndpoint = reconTestInjector.getInstance(NSSummaryEndpoint.class); + + scenario.populateAndReprocess(reconNamespaceSummaryManager, + reconOMMetadataManager, conf); + } + + // --------------------------------------------------------------------------- + // Tests common to every layout + // --------------------------------------------------------------------------- + + @Test + public void testUtility() { + String[] names = EntityHandler.parseRequestPath( + NSSummaryTestScenario.TEST_PATH_UTILITY); + assertArrayEquals(NSSummaryTestScenario.TEST_NAMES, names); + assertEquals(NSSummaryTestScenario.TEST_KEY_NAMES, + BucketHandler.getKeyName(names)); + assertEquals(NSSummaryTestScenario.TEST_PATH_UTILITY, + BucketHandler.buildSubpath(NSSummaryTestScenario.PARENT_DIR, + "file1.txt")); + } + + @Test + public void testGetBasicInfoRoot() throws Exception { + scenario.assertBasicInfoRoot(nsSummaryEndpoint, reconOMMetadataManager); + } + + @Test + public void testGetBasicInfoVol() throws Exception { + scenario.assertBasicInfoVolume(nsSummaryEndpoint); + } + + @Test + public void testGetBasicInfoBucketOne() throws Exception { + scenario.assertBasicInfoBucketOne(nsSummaryEndpoint); + } + + @Test + public void testGetBasicInfoBucketTwo() throws Exception { + scenario.assertBasicInfoBucketTwo(nsSummaryEndpoint); + } + + @Test + public void testGetBasicInfoNoPath() throws Exception { + assertBasicInfoNoPath(nsSummaryEndpoint, NSSummaryTestScenario.INVALID_PATH); + } + + @Test + public void testGetBasicInfoKey() throws Exception { + assertBasicInfoKey(nsSummaryEndpoint, NSSummaryTestScenario.KEY_PATH, + 2 * KB + 1); + } + + @Test + public void testDiskUsageRoot() throws Exception { + scenario.assertDiskUsageRoot(nsSummaryEndpoint); + } + + @Test + public void testDiskUsageVolume() throws Exception { + scenario.assertDiskUsageVolume(nsSummaryEndpoint); + } + + @Test + public void testDiskUsageKey() throws Exception { + scenario.assertDiskUsageKey(nsSummaryEndpoint); + } + + @Test + public void testDiskUsageUnknown() throws Exception { + Response invalidResponse = nsSummaryEndpoint.getDiskUsage( + NSSummaryTestScenario.INVALID_PATH, false, false, false); + DUResponse invalidObj = (DUResponse) invalidResponse.getEntity(); + assertEquals(ResponseStatus.PATH_NOT_FOUND, invalidObj.getStatus()); + } + + @Test + public void testDiskUsageWithReplication() throws Exception { + scenario.writeMultiBlockKey(reconOMMetadataManager); + DUResponse replicaDUResponse = getDiskUsage(nsSummaryEndpoint, + scenario.multiBlockKeyPath(), true); + assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); + assertEquals(scenario.multiBlockKeyReplicatedSize(), + replicaDUResponse.getSizeWithReplica()); + } + + @Test + public void testDataSizeUnderRootWithReplication() throws IOException { + scenario.assertDataSizeUnderRootWithReplication(nsSummaryEndpoint); + } + + @Test + public void testDataSizeUnderVolWithReplication() throws IOException { + scenario.assertDataSizeUnderVolWithReplication(nsSummaryEndpoint); + } + + @Test + public void testDataSizeUnderKeyWithReplication() throws IOException { + scenario.assertDataSizeUnderKeyWithReplication(nsSummaryEndpoint); + } + + @Test + public void testQuotaUsage() throws Exception { + scenario.assertQuotaUsage(nsSummaryEndpoint); + } + + @Test + public void testFileSizeDist() throws Exception { + scenario.assertFileSizeDist(nsSummaryEndpoint); + } + + @Test + public void testConstructFullPath() throws IOException { + scenario.verifyConstructFullPath(reconOMMetadataManager, + reconNamespaceSummaryManager); + } + + @Test + public void testConstructFullPathWithNegativeParentIdTriggersRebuild() + throws IOException { + long dirOneObjectId = 1L; + ReconNamespaceSummaryManager mockSummaryManager = + mock(ReconNamespaceSummaryManager.class); + NSSummary dir1Summary = new NSSummary(); + dir1Summary.setParentId(-1); + when(mockSummaryManager.getNSSummary(dirOneObjectId)) + .thenReturn(dir1Summary); + + OmKeyInfo keyInfo = new OmKeyInfo.Builder() + .setKeyName("file2") + .setVolumeName("vol") + .setBucketName("bucket1") + .setObjectID(2L) + .setParentObjectID(dirOneObjectId) + .build(); + + assertEquals("", + ReconUtils.constructFullPath(keyInfo, mockSummaryManager), + "Should return empty string when NSSummary has negative parentId"); + } + + // --------------------------------------------------------------------------- + // Tree-only tests (FSO, Legacy) + // --------------------------------------------------------------------------- + + @Test + public void testGetBasicInfoDir() throws Exception { + assumeTrue(scenario.hasDirectories()); + treeScenario().assertBasicInfoDir(nsSummaryEndpoint); + } + + @Test + public void testDiskUsageBucket() throws Exception { + assumeTrue(scenario.hasDirectories()); + treeScenario().assertDiskUsageBucket(nsSummaryEndpoint); + } + + @Test + public void testDiskUsageDir() throws Exception { + assumeTrue(scenario.hasDirectories()); + treeScenario().assertDiskUsageDir(nsSummaryEndpoint); + } + + @Test + public void testDataSizeUnderBucketWithReplication() throws IOException { + assumeTrue(scenario.hasDirectories()); + treeScenario().assertDataSizeUnderBucketWithReplication(nsSummaryEndpoint); + } + + @Test + public void testDataSizeUnderDirWithReplication() throws IOException { + assumeTrue(scenario.hasDirectories()); + treeScenario().assertDataSizeUnderDirWithReplication(nsSummaryEndpoint); + } + + @Test + public void testReplicatedSizePropagationUpwards() throws IOException { + assumeTrue(scenario.hasDirectories()); + treeScenario().assertReplicatedSizePropagation(nsSummaryEndpoint); + } + + @Test + public void testDataSizeUnderVolumeWithRatisReplication() throws IOException { + assumeTrue(scenario.hasVolumeThree()); + treeScenario().assertRatisReplicationUnderVolumeThree(nsSummaryEndpoint); + } + + @Test + public void testDataSizeUnderBucketWithRatisReplication() throws IOException { + assumeTrue(scenario.hasVolumeThree()); + treeScenario().assertRatisReplicationUnderBucketFive(nsSummaryEndpoint); + } + + @Test + public void testDataSizeUnderDirWithRatisReplication() throws IOException { + assumeTrue(scenario.hasVolumeThree()); + treeScenario().assertRatisReplicationUnderDirSix(nsSummaryEndpoint); + } + + // --------------------------------------------------------------------------- + // Flat-only tests (OBS + Legacy) + // --------------------------------------------------------------------------- + + @Test + public void testGetBasicInfoVolTwo() throws Exception { + assumeTrue(scenario.isFlatLayout()); + flatScenario().assertBasicInfoVolTwo(nsSummaryEndpoint); + } + + @Test + public void testGetBasicInfoBucketThree() throws Exception { + assumeTrue(scenario.isFlatLayout()); + flatScenario().assertBasicInfoBucketThree(nsSummaryEndpoint); + } + + @Test + public void testGetBasicInfoBucketFour() throws Exception { + assumeTrue(scenario.isFlatLayout()); + flatScenario().assertBasicInfoBucketFour(nsSummaryEndpoint); + } + + @Test + public void testDiskUsageVolTwo() throws Exception { + assumeTrue(scenario.isFlatLayout()); + flatScenario().assertDiskUsageVolTwo(nsSummaryEndpoint); + } + + @Test + public void testDiskUsageBucketOne() throws Exception { + assumeTrue(scenario.isFlatLayout()); + flatScenario().assertDiskUsageBucketOne(nsSummaryEndpoint); + } + + @Test + public void testDiskUsageBucketTwo() throws Exception { + assumeTrue(scenario.isFlatLayout()); + flatScenario().assertDiskUsageBucketTwo(nsSummaryEndpoint); + } + + @Test + public void testDiskUsageBucketThree() throws Exception { + assumeTrue(scenario.isFlatLayout()); + flatScenario().assertDiskUsageBucketThree(nsSummaryEndpoint); + } + + @Test + public void testDiskUsageKeys() throws Exception { + assumeTrue(scenario.isFlatLayout()); + flatScenario().assertDiskUsageKeys(nsSummaryEndpoint); + } + + @Test + public void testDataSizeUnderBucketOneWithReplication() throws IOException { + assumeTrue(scenario.isFlatLayout()); + flatScenario().assertDataSizeUnderBucketOneWithReplication( + nsSummaryEndpoint); + } + + @Test + public void testDataSizeUnderBucketThreeWithReplication() throws IOException { + assumeTrue(scenario.isFlatLayout()); + flatScenario().assertDataSizeUnderBucketThreeWithReplication( + nsSummaryEndpoint); + } + + @Test + public void testNormalizePathUptoBucket() { + assumeTrue(scenario.isFlatLayout()); + flatScenario().assertNormalizePathUptoBucket(); + } + + private AbstractTreeNSSummaryScenario treeScenario() { + return (AbstractTreeNSSummaryScenario) scenario; + } + + private FlatNSSummaryScenario flatScenario() { + return (FlatNSSummaryScenario) scenario; + } +} diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java deleted file mode 100644 index 320494b3fc82..000000000000 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java +++ /dev/null @@ -1,1577 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.recon.api; - -import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS; -import static org.apache.hadoop.ozone.om.helpers.QuotaUtil.getReplicatedSize; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getMockOzoneManagerServiceProviderWithFSO; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDirToOm; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm; -import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import javax.ws.rs.core.Response; -import org.apache.hadoop.hdds.client.BlockID; -import org.apache.hadoop.hdds.client.RatisReplicationConfig; -import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State; -import org.apache.hadoop.hdds.scm.container.ContainerID; -import org.apache.hadoop.hdds.scm.container.ContainerManager; -import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; -import org.apache.hadoop.hdds.scm.container.ContainerReplica; -import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat; -import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; -import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; -import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; -import org.apache.hadoop.ozone.om.helpers.QuotaUtil; -import org.apache.hadoop.ozone.recon.ReconConstants; -import org.apache.hadoop.ozone.recon.ReconTestInjector; -import org.apache.hadoop.ozone.recon.ReconUtils; -import org.apache.hadoop.ozone.recon.api.handlers.BucketHandler; -import org.apache.hadoop.ozone.recon.api.handlers.EntityHandler; -import org.apache.hadoop.ozone.recon.api.types.DUResponse; -import org.apache.hadoop.ozone.recon.api.types.FileSizeDistributionResponse; -import org.apache.hadoop.ozone.recon.api.types.NSSummary; -import org.apache.hadoop.ozone.recon.api.types.QuotaUsageResponse; -import org.apache.hadoop.ozone.recon.api.types.ResponseStatus; -import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; -import org.apache.hadoop.ozone.recon.scm.ReconNodeManager; -import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade; -import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; -import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; -import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; -import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; -import org.apache.hadoop.ozone.recon.tasks.NSSummaryTaskWithFSO; -import org.apache.hadoop.util.Time; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -/** - * Test for NSSummary REST APIs with FSO. - * We tested on a mini file system with the following setting: - * vol - * / \ - * bucket1 bucket2 - * / \ / \ - * file1 dir1 file4 file5 - * / \ \ - * dir2 dir3 dir4 - * / \ \ - * file2 file3 file6 - * ---------------------------------------- - * vol2 - * / \ - * bucket3 bucket4 - * / \ / - * file8 dir5 file11 - * / \ - * file9 file10 - * ---------------------------------------- - * vol3 - * | - * bucket5 - * / \ - * file12 dir6 - * / \ - * file13 dir7 - * / - * file14 - * This is a test for the Rest APIs only. We have tested NSSummaryTask before, - * so there is no need to test process() on DB's updates - */ -public class TestNSSummaryEndpointWithFSO extends NSSummaryTests { - @TempDir - private Path temporaryFolder; - - private ReconOMMetadataManager reconOMMetadataManager; - private ReconNamespaceSummaryManager reconNamespaceSummaryManager; - private NSSummaryEndpoint nsSummaryEndpoint; - - private static final String TEST_PATH_UTILITY = - "/vol1/buck1/a/b/c/d/e/file1.txt"; - private static final String PARENT_DIR = "vol1/buck1/a/b/c/d/e"; - private static final String[] TEST_NAMES = - new String[]{"vol1", "buck1", "a", "b", "c", "d", "e", "file1.txt"}; - private static final String TEST_KEY_NAMES = "a/b/c/d/e/file1.txt"; - - // Object names in FSO-enabled format - private static final String VOL = "vol"; - private static final String VOL_TWO = "vol2"; - private static final String VOL_THREE = "vol3"; - private static final String BUCKET_ONE = "bucket1"; - private static final String BUCKET_TWO = "bucket2"; - private static final String BUCKET_THREE = "bucket3"; - private static final String BUCKET_FOUR = "bucket4"; - private static final String BUCKET_FIVE = "bucket5"; - private static final String KEY_ONE = "file1"; - private static final String KEY_TWO = "dir1/dir2/file2"; - private static final String KEY_THREE = "dir1/dir3/file3"; - private static final String KEY_FOUR = "file4"; - private static final String KEY_FIVE = "file5"; - private static final String KEY_SIX = "dir1/dir4/file6"; - private static final String KEY_SEVEN = "dir1/file7"; - private static final String KEY_EIGHT = "file8"; - private static final String KEY_NINE = "dir5/file9"; - private static final String KEY_TEN = "dir5/file10"; - private static final String KEY_ELEVEN = "file11"; - private static final String MULTI_BLOCK_KEY = "dir1/file7"; - private static final String MULTI_BLOCK_FILE = "file7"; - private static final String KEY_TWELVE = "file12"; - private static final String KEY_THIRTEEN = "dir6/file13"; - private static final String KEY_FOURTEEN = "dir6/dir7/file14"; - - private static final String FILE_ONE = "file1"; - private static final String FILE_TWO = "file2"; - private static final String FILE_THREE = "file3"; - private static final String FILE_FOUR = "file4"; - private static final String FILE_FIVE = "file5"; - private static final String FILE_SIX = "file6"; - private static final String FILE_SEVEN = "file7"; - private static final String FILE_EIGHT = "file8"; - private static final String FILE_NINE = "file9"; - private static final String FILE_TEN = "file10"; - private static final String FILE_ELEVEN = "file11"; - private static final String FILE_TWELVE = "file12"; - private static final String FILE_THIRTEEN = "file13"; - private static final String FILE_FOURTEEN = "file14"; - - private static final String DIR_ONE = "dir1"; - private static final String DIR_TWO = "dir2"; - private static final String DIR_THREE = "dir3"; - private static final String DIR_FOUR = "dir4"; - private static final String DIR_FIVE = "dir5"; - private static final String DIR_SIX = "dir6"; - private static final String DIR_SEVEN = "dir7"; - // objects IDs - private static final long VOL_OBJECT_ID = 0L; - private static final long BUCKET_ONE_OBJECT_ID = 1L; - private static final long BUCKET_TWO_OBJECT_ID = 2L; - private static final long KEY_ONE_OBJECT_ID = 3L; - private static final long DIR_ONE_OBJECT_ID = 4L; - private static final long KEY_TWO_OBJECT_ID = 5L; - private static final long KEY_FOUR_OBJECT_ID = 6L; - private static final long DIR_TWO_OBJECT_ID = 7L; - private static final long KEY_THREE_OBJECT_ID = 8L; - private static final long KEY_FIVE_OBJECT_ID = 9L; - private static final long KEY_SIX_OBJECT_ID = 10L; - private static final long DIR_THREE_OBJECT_ID = 11L; - private static final long DIR_FOUR_OBJECT_ID = 12L; - private static final long MULTI_BLOCK_KEY_OBJECT_ID = 13L; - private static final long KEY_SEVEN_OBJECT_ID = 13L; - private static final long VOL_TWO_OBJECT_ID = 14L; - private static final long BUCKET_THREE_OBJECT_ID = 15L; - private static final long BUCKET_FOUR_OBJECT_ID = 16L; - private static final long KEY_EIGHT_OBJECT_ID = 17L; - private static final long DIR_FIVE_OBJECT_ID = 18L; - private static final long KEY_NINE_OBJECT_ID = 19L; - private static final long KEY_TEN_OBJECT_ID = 20L; - private static final long KEY_ELEVEN_OBJECT_ID = 21L; - private static final long VOL_THREE_OBJECT_ID = 22L; - private static final long DIR_SIX_OBJECT_ID = 23L; - private static final long DIR_SEVEN_OBJECT_ID = 24L; - private static final long FILE_TWELVE_OBJECT_ID = 25L; - private static final long FILE_THIRTEEN_OBJECT_ID = 26L; - private static final long FILE_FOURTEEN_OBJECT_ID = 27L; - private static final long BUCKET_FIVE_OBJECT_ID = 28L; - - // container IDs - private static final long CONTAINER_ONE_ID = 1L; - private static final long CONTAINER_TWO_ID = 2L; - private static final long CONTAINER_THREE_ID = 3L; - private static final long CONTAINER_FOUR_ID = 4L; - private static final long CONTAINER_FIVE_ID = 5L; - private static final long CONTAINER_SIX_ID = 6L; - - // replication factors - private static final int CONTAINER_ONE_REPLICA_COUNT = 3; - private static final int CONTAINER_TWO_REPLICA_COUNT = 2; - private static final int CONTAINER_THREE_REPLICA_COUNT = 4; - private static final int CONTAINER_FOUR_REPLICA_COUNT = 5; - private static final int CONTAINER_FIVE_REPLICA_COUNT = 2; - private static final int CONTAINER_SIX_REPLICA_COUNT = 3; - - // block lengths - private static final long BLOCK_ONE_LENGTH = 1000L; - private static final long BLOCK_TWO_LENGTH = 2000L; - private static final long BLOCK_THREE_LENGTH = 3000L; - private static final long BLOCK_FOUR_LENGTH = 4000L; - private static final long BLOCK_FIVE_LENGTH = 5000L; - private static final long BLOCK_SIX_LENGTH = 6000L; - - // data size in bytes - private static final long KEY_ONE_SIZE = 500L; // bin 0 - private static final long KEY_TWO_SIZE = OzoneConsts.KB + 1; // bin 1 - private static final long KEY_THREE_SIZE = 4 * OzoneConsts.KB + 1; // bin 3 - private static final long KEY_FOUR_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 - private static final long KEY_FIVE_SIZE = 100L; // bin 0 - private static final long KEY_SIX_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 - private static final long KEY_SEVEN_SIZE = 4 * OzoneConsts.KB + 1; - private static final long KEY_EIGHT_SIZE = OzoneConsts.KB + 1; // bin 1 - private static final long KEY_NINE_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 - private static final long KEY_TEN_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 - private static final long KEY_ELEVEN_SIZE = OzoneConsts.KB + 1; // bin 1 - private static final long KEY_TWELVE_SIZE = OzoneConsts.KB; - private static final long KEY_THIRTEEN_SIZE = OzoneConsts.KB; - private static final long KEY_FOURTEEN_SIZE = OzoneConsts.KB; - - private static final long FILE1_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_ONE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE2_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_TWO_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE3_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_THREE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE4_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_FOUR_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE5_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_FIVE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE6_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_SIX_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE7_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_SEVEN_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE8_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_EIGHT_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE9_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_NINE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE10_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_TEN_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE11_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_ELEVEN_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE12_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_TWELVE_SIZE, RatisReplicationConfig.getInstance(THREE)); - private static final long FILE13_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_THIRTEEN_SIZE, RatisReplicationConfig.getInstance(THREE)); - private static final long FILE14_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_FOURTEEN_SIZE, RatisReplicationConfig.getInstance(THREE)); - private static final long MULTI_BLOCK_KEY_SIZE_WITH_REPLICA - = FILE7_SIZE_WITH_REPLICA; - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_ROOT - = FILE1_SIZE_WITH_REPLICA - + FILE2_SIZE_WITH_REPLICA - + FILE3_SIZE_WITH_REPLICA - + FILE4_SIZE_WITH_REPLICA - + FILE5_SIZE_WITH_REPLICA - + FILE6_SIZE_WITH_REPLICA - + FILE7_SIZE_WITH_REPLICA - + FILE8_SIZE_WITH_REPLICA - + FILE9_SIZE_WITH_REPLICA - + FILE10_SIZE_WITH_REPLICA - + FILE11_SIZE_WITH_REPLICA - + FILE12_SIZE_WITH_REPLICA - + FILE13_SIZE_WITH_REPLICA - + FILE14_SIZE_WITH_REPLICA; - - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_VOL - = FILE1_SIZE_WITH_REPLICA - + FILE2_SIZE_WITH_REPLICA - + FILE3_SIZE_WITH_REPLICA - + FILE4_SIZE_WITH_REPLICA - + FILE5_SIZE_WITH_REPLICA - + FILE6_SIZE_WITH_REPLICA - + FILE7_SIZE_WITH_REPLICA; - - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET1 - = FILE1_SIZE_WITH_REPLICA - + FILE2_SIZE_WITH_REPLICA - + FILE3_SIZE_WITH_REPLICA - + FILE6_SIZE_WITH_REPLICA - + FILE7_SIZE_WITH_REPLICA; - - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_DIR1 - = FILE2_SIZE_WITH_REPLICA - + FILE3_SIZE_WITH_REPLICA - + FILE6_SIZE_WITH_REPLICA - + FILE7_SIZE_WITH_REPLICA; - - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_DIR2 - = FILE2_SIZE_WITH_REPLICA; - - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_KEY - = FILE4_SIZE_WITH_REPLICA; - - // quota in bytes - private static final long ROOT_QUOTA = 2 * (2 * OzoneConsts.MB); - private static final long VOL_QUOTA = 2 * OzoneConsts.MB; - private static final long VOL_TWO_QUOTA = 2 * OzoneConsts.MB; - private static final long VOL_THREE_QUOTA = 2 * OzoneConsts.MB; - private static final long BUCKET_ONE_QUOTA = OzoneConsts.MB; - private static final long BUCKET_TWO_QUOTA = OzoneConsts.MB; - private static final long BUCKET_THREE_QUOTA = OzoneConsts.MB; - private static final long BUCKET_FOUR_QUOTA = OzoneConsts.MB; - private static final long BUCKET_FIVE_QUOTA = OzoneConsts.MB; - - // mock client's path requests - private static final String TEST_USER = "TestUser"; - private static final String ROOT_PATH = "/"; - private static final String VOL_PATH = "/vol"; - private static final String VOL_TWO_PATH = "/vol2"; - private static final String BUCKET_ONE_PATH = "/vol/bucket1"; - private static final String BUCKET_TWO_PATH = "/vol/bucket2"; - private static final String DIR_ONE_PATH = "/vol/bucket1/dir1"; - private static final String DIR_TWO_PATH = "/vol/bucket1/dir1/dir2"; - private static final String DIR_THREE_PATH = "/vol/bucket1/dir1/dir3"; - private static final String DIR_FOUR_PATH = "/vol/bucket1/dir1/dir4"; - private static final String KEY_PATH = "/vol/bucket2/file4"; - private static final String MULTI_BLOCK_KEY_PATH = "/vol/bucket1/dir1/file7"; - private static final String INVALID_PATH = "/vol/path/not/found"; - private static final String VOL_THREE_PATH = "/vol3"; - - // some expected answers - private static final long ROOT_DATA_SIZE = KEY_ONE_SIZE + KEY_TWO_SIZE + - KEY_THREE_SIZE + KEY_FOUR_SIZE + KEY_FIVE_SIZE + KEY_SIX_SIZE + KEY_SEVEN_SIZE + - KEY_EIGHT_SIZE + KEY_NINE_SIZE + KEY_TEN_SIZE + KEY_ELEVEN_SIZE + - FILE12_SIZE_WITH_REPLICA + FILE13_SIZE_WITH_REPLICA + FILE14_SIZE_WITH_REPLICA; - - private static final long VOL_DATA_SIZE = KEY_ONE_SIZE + KEY_TWO_SIZE + - KEY_THREE_SIZE + KEY_FOUR_SIZE + KEY_FIVE_SIZE + KEY_SIX_SIZE + KEY_SEVEN_SIZE; - - private static final long VOL_TWO_DATA_SIZE = - KEY_EIGHT_SIZE + KEY_NINE_SIZE + KEY_TEN_SIZE + KEY_ELEVEN_SIZE; - - private static final long BUCKET_ONE_DATA_SIZE = KEY_ONE_SIZE + KEY_TWO_SIZE + - KEY_THREE_SIZE + KEY_SIX_SIZE + KEY_SEVEN_SIZE; - - private static final long BUCKET_TWO_DATA_SIZE = - KEY_FOUR_SIZE + KEY_FIVE_SIZE; - - private static final long DIR_ONE_DATA_SIZE = KEY_TWO_SIZE + - KEY_THREE_SIZE + KEY_SIX_SIZE + KEY_SEVEN_SIZE; - - @BeforeEach - public void setUp() throws Exception { - OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); - ozoneConfiguration.setLong(OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD, - 10); - OMMetadataManager omMetadataManager = initializeNewOmMetadataManager( - Files.createDirectory(temporaryFolder.resolve("JunitOmDBDir")) - .toFile()); - OzoneManagerServiceProviderImpl ozoneManagerServiceProvider = - getMockOzoneManagerServiceProviderWithFSO(); - reconOMMetadataManager = getTestReconOmMetadataManager(omMetadataManager, - Files.createDirectory(temporaryFolder.resolve("OmMetataDir")).toFile()); - - ReconTestInjector reconTestInjector = - new ReconTestInjector.Builder(temporaryFolder.toFile()) - .withReconOm(reconOMMetadataManager) - .withOmServiceProvider(ozoneManagerServiceProvider) - .withReconSqlDb() - .withContainerDB() - .addBinding(OzoneStorageContainerManager.class, - getMockReconSCM()) - .addBinding(StorageContainerServiceProvider.class, - mock(StorageContainerServiceProviderImpl.class)) - .addBinding(NSSummaryEndpoint.class) - .build(); - this.reconNamespaceSummaryManager = - reconTestInjector.getInstance(ReconNamespaceSummaryManager.class); - nsSummaryEndpoint = reconTestInjector.getInstance(NSSummaryEndpoint.class); - - // populate OM DB and reprocess into Recon RocksDB - populateOMDB(); - populateVolumeThree(); - setUpMultiBlockReplicatedKeys(); - NSSummaryTaskWithFSO nSSummaryTaskWithFso = - new NSSummaryTaskWithFSO(reconNamespaceSummaryManager, reconOMMetadataManager, 10, 5, 20, 2000); - nSSummaryTaskWithFso.reprocessWithFSO(reconOMMetadataManager); - } - - @Test - public void testUtility() { - String[] names = EntityHandler.parseRequestPath(TEST_PATH_UTILITY); - assertArrayEquals(TEST_NAMES, names); - String keyName = BucketHandler.getKeyName(names); - assertEquals(TEST_KEY_NAMES, keyName); - String subpath = BucketHandler.buildSubpath(PARENT_DIR, "file1.txt"); - assertEquals(TEST_PATH_UTILITY, subpath); - } - - @Test - public void testGetBasicInfoRoot() throws Exception { - // Test root basics - testNSSummaryBasicInfoRoot( - nsSummaryEndpoint, reconOMMetadataManager); - } - - @Test - public void testGetBasicInfoVol() throws Exception { - // Test volume basics - testNSSummaryBasicInfoVolume(nsSummaryEndpoint); - } - - @Test - public void testGetBasicInfoBucketOne() throws Exception { - // Test bucket 1's basics - testNSSummaryBasicInfoBucketOne( - BucketLayout.FILE_SYSTEM_OPTIMIZED, - nsSummaryEndpoint); - } - - @Test - public void testGetBasicInfoBucketTwo() throws Exception { - // Test bucket 2's basics - testNSSummaryBasicInfoBucketTwo( - BucketLayout.FILE_SYSTEM_OPTIMIZED, - nsSummaryEndpoint); - } - - @Test - public void testGetBasicInfoDir() throws Exception { - // Test intermediate directory basics - testNSSummaryBasicInfoDir(nsSummaryEndpoint); - } - - @Test - public void testGetBasicInfoNoPath() throws Exception { - // Test invalid path - testNSSummaryBasicInfoNoPath(nsSummaryEndpoint); - } - - @Test - public void testGetBasicInfoKey() throws Exception { - // Test key - testNSSummaryBasicInfoKey(nsSummaryEndpoint); - } - - @Test - public void testDiskUsageRoot() throws Exception { - // root level DU - Response rootResponse = nsSummaryEndpoint.getDiskUsage(ROOT_PATH, - false, false, false); - DUResponse duRootRes = (DUResponse) rootResponse.getEntity(); - assertEquals(3, duRootRes.getCount()); - List duRootData = duRootRes.getDuData(); - // sort based on subpath - Collections.sort(duRootData, - Comparator.comparing(DUResponse.DiskUsage::getSubpath)); - DUResponse.DiskUsage duVol1 = duRootData.get(0); - DUResponse.DiskUsage duVol2 = duRootData.get(1); - assertEquals(VOL_PATH, duVol1.getSubpath()); - assertEquals(VOL_TWO_PATH, duVol2.getSubpath()); - assertEquals(VOL_DATA_SIZE, duVol1.getSize()); - assertEquals(VOL_TWO_DATA_SIZE, duVol2.getSize()); - } - - @Test - public void testDiskUsageVolume() throws Exception { - // volume level DU - Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_PATH, - false, false, false); - DUResponse duVolRes = (DUResponse) volResponse.getEntity(); - assertEquals(2, duVolRes.getCount()); - List duData = duVolRes.getDuData(); - // sort based on subpath - Collections.sort(duData, - Comparator.comparing(DUResponse.DiskUsage::getSubpath)); - DUResponse.DiskUsage duBucket1 = duData.get(0); - DUResponse.DiskUsage duBucket2 = duData.get(1); - assertEquals(BUCKET_ONE_PATH, duBucket1.getSubpath()); - assertEquals(BUCKET_TWO_PATH, duBucket2.getSubpath()); - assertEquals(BUCKET_ONE_DATA_SIZE, duBucket1.getSize()); - assertEquals(BUCKET_TWO_DATA_SIZE, duBucket2.getSize()); - - } - - @Test - public void testDiskUsageBucket() throws Exception { - // bucket level DU - Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_ONE_PATH, - false, false, false); - DUResponse duBucketResponse = (DUResponse) bucketResponse.getEntity(); - assertEquals(1, duBucketResponse.getCount()); - DUResponse.DiskUsage duDir1 = duBucketResponse.getDuData().get(0); - assertEquals(DIR_ONE_PATH, duDir1.getSubpath()); - assertEquals(DIR_ONE_DATA_SIZE, duDir1.getSize()); - - } - - @Test - public void testDiskUsageDir() throws Exception { - // dir level DU - Response dirResponse = nsSummaryEndpoint.getDiskUsage(DIR_ONE_PATH, - false, false, false); - DUResponse duDirReponse = (DUResponse) dirResponse.getEntity(); - assertEquals(3, duDirReponse.getCount()); - List duSubDir = duDirReponse.getDuData(); - Collections.sort(duSubDir, - Comparator.comparing(DUResponse.DiskUsage::getSubpath)); - DUResponse.DiskUsage duDir2 = duSubDir.get(0); - DUResponse.DiskUsage duDir3 = duSubDir.get(1); - DUResponse.DiskUsage duDir4 = duSubDir.get(2); - assertEquals(DIR_TWO_PATH, duDir2.getSubpath()); - assertEquals(KEY_TWO_SIZE, duDir2.getSize()); - - assertEquals(DIR_THREE_PATH, duDir3.getSubpath()); - assertEquals(KEY_THREE_SIZE, duDir3.getSize()); - - assertEquals(DIR_FOUR_PATH, duDir4.getSubpath()); - assertEquals(KEY_SIX_SIZE, duDir4.getSize()); - - } - - @Test - public void testDiskUsageKey() throws Exception { - // key level DU - Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_PATH, - false, false, false); - DUResponse keyObj = (DUResponse) keyResponse.getEntity(); - assertEquals(0, keyObj.getCount()); - assertEquals(KEY_FOUR_SIZE, keyObj.getSize()); - - } - - @Test - public void testDiskUsageUnknown() throws Exception { - // invalid path check - Response invalidResponse = nsSummaryEndpoint.getDiskUsage(INVALID_PATH, - false, false, false); - DUResponse invalidObj = (DUResponse) invalidResponse.getEntity(); - assertEquals(ResponseStatus.PATH_NOT_FOUND, - invalidObj.getStatus()); - } - - @Test - public void testDiskUsageWithReplication() throws Exception { - setUpMultiBlockKey(); - Response keyResponse = nsSummaryEndpoint.getDiskUsage(MULTI_BLOCK_KEY_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_KEY_SIZE_WITH_REPLICA, - replicaDUResponse.getSizeWithReplica()); - } - - @Test - public void testDataSizeUnderRootWithReplication() throws IOException { - // withReplica is true - Response rootResponse = nsSummaryEndpoint.getDiskUsage(ROOT_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) rootResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_ROOT, - replicaDUResponse.getSizeWithReplica()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_VOL, - replicaDUResponse.getDuData().get(0).getSizeWithReplica()); - - } - - @Test - public void testDataSizeUnderVolWithReplication() throws IOException { - Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) volResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_VOL, - replicaDUResponse.getSizeWithReplica()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET1, - replicaDUResponse.getDuData().get(0).getSizeWithReplica()); - } - - @Test - public void testDataSizeUnderBucketWithReplication() throws IOException { - Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_ONE_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) bucketResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET1, - replicaDUResponse.getSizeWithReplica()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_DIR1, - replicaDUResponse.getDuData().get(0).getSizeWithReplica()); - } - - @Test - public void testReplicatedSizePropagationUpwards() throws IOException { - // Test that replicated size propagates correctly from files up through the directory hierarchy - - // Get disk usage for individual files first to establish baseline - DUResponse file2Response = getDiskUsageResponse(DIR_TWO_PATH + "/file2"); - DUResponse file3Response = getDiskUsageResponse(DIR_THREE_PATH + "/file3"); - DUResponse file6Response = getDiskUsageResponse(DIR_FOUR_PATH + "/file6"); - DUResponse file7Response = getDiskUsageResponse(DIR_ONE_PATH + "/file7"); - - // Verify individual file replicated sizes - assertEquals(FILE2_SIZE_WITH_REPLICA, file2Response.getSizeWithReplica()); - assertEquals(FILE3_SIZE_WITH_REPLICA, file3Response.getSizeWithReplica()); - assertEquals(FILE6_SIZE_WITH_REPLICA, file6Response.getSizeWithReplica()); - assertEquals(FILE7_SIZE_WITH_REPLICA, file7Response.getSizeWithReplica()); - - // Test dir2 (contains only file2) - DUResponse dir2Response = getDiskUsageResponse(DIR_TWO_PATH); - assertEquals(FILE2_SIZE_WITH_REPLICA, dir2Response.getSizeWithReplica()); - - // Test dir3 (contains only file3) - DUResponse dir3Response = getDiskUsageResponse(DIR_THREE_PATH); - assertEquals(FILE3_SIZE_WITH_REPLICA, dir3Response.getSizeWithReplica()); - - // Test dir4 (contains only file6) - DUResponse dir4Response = getDiskUsageResponse(DIR_FOUR_PATH); - assertEquals(FILE6_SIZE_WITH_REPLICA, dir4Response.getSizeWithReplica()); - - // Test dir1 (contains file7 directly + dir2, dir3, dir4 contents) - DUResponse dir1Response = getDiskUsageResponse(DIR_ONE_PATH); - long expectedDir1ReplicatedSize = FILE2_SIZE_WITH_REPLICA + FILE3_SIZE_WITH_REPLICA + - FILE6_SIZE_WITH_REPLICA + FILE7_SIZE_WITH_REPLICA; - assertEquals(expectedDir1ReplicatedSize, dir1Response.getSizeWithReplica()); - - // Test bucket1 (contains file1 directly + all dir1 contents) - DUResponse bucket1Response = getDiskUsageResponse(BUCKET_ONE_PATH); - long expectedBucket1ReplicatedSize = FILE1_SIZE_WITH_REPLICA + expectedDir1ReplicatedSize; - assertEquals(expectedBucket1ReplicatedSize, bucket1Response.getSizeWithReplica()); - - // Test bucket2 (contains file4 and file5) - DUResponse bucket2Response = getDiskUsageResponse(BUCKET_TWO_PATH); - long expectedBucket2ReplicatedSize = FILE4_SIZE_WITH_REPLICA + FILE5_SIZE_WITH_REPLICA; - assertEquals(expectedBucket2ReplicatedSize, bucket2Response.getSizeWithReplica()); - - // Test vol (contains bucket1 + bucket2) - DUResponse volResponse = getDiskUsageResponse(VOL_PATH); - long expectedVolReplicatedSize = expectedBucket1ReplicatedSize + expectedBucket2ReplicatedSize; - assertEquals(expectedVolReplicatedSize, volResponse.getSizeWithReplica()); - - // Test root (contains vol + vol2) - DUResponse rootResponse = getDiskUsageResponse(ROOT_PATH); - long expectedVol2ReplicatedSize = FILE8_SIZE_WITH_REPLICA + FILE9_SIZE_WITH_REPLICA + - FILE10_SIZE_WITH_REPLICA + FILE11_SIZE_WITH_REPLICA; - long vol3TotalSize = FILE12_SIZE_WITH_REPLICA + FILE13_SIZE_WITH_REPLICA + FILE14_SIZE_WITH_REPLICA; - long expectedRootReplicatedSize = expectedVolReplicatedSize + expectedVol2ReplicatedSize + vol3TotalSize; - assertEquals(expectedRootReplicatedSize, rootResponse.getSizeWithReplica()); - } - - private DUResponse getDiskUsageResponse(String path) throws IOException { - Response response = nsSummaryEndpoint.getDiskUsage(path, false, true, false); - return (DUResponse) response.getEntity(); - } - - /** - * When calculating DU under dir1 - * there are 3 keys, file2, file3, file6. - * There is one direct key, file7. - * @throws IOException - */ - @Test - public void testDataSizeUnderDirWithReplication() throws IOException { - Response dir1Response = nsSummaryEndpoint.getDiskUsage(DIR_ONE_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) dir1Response.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_DIR1, - replicaDUResponse.getSizeWithReplica()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_DIR2, - replicaDUResponse.getDuData().get(0).getSizeWithReplica()); - } - - @Test - public void testDataSizeUnderKeyWithReplication() throws IOException { - Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_KEY, - replicaDUResponse.getSizeWithReplica()); - } - - @Test - public void testQuotaUsage() throws Exception { - // root level quota usage - Response rootResponse = nsSummaryEndpoint.getQuotaUsage(ROOT_PATH); - QuotaUsageResponse quRootRes = - (QuotaUsageResponse) rootResponse.getEntity(); - assertEquals(ROOT_QUOTA, quRootRes.getQuota()); - assertEquals(ROOT_DATA_SIZE, quRootRes.getQuotaUsed()); - - // volume level quota usage - Response volResponse = nsSummaryEndpoint.getQuotaUsage(VOL_PATH); - QuotaUsageResponse quVolRes = (QuotaUsageResponse) volResponse.getEntity(); - assertEquals(VOL_QUOTA, quVolRes.getQuota()); - assertEquals(VOL_DATA_SIZE, quVolRes.getQuotaUsed()); - - // bucket level quota usage - Response bucketRes = nsSummaryEndpoint.getQuotaUsage(BUCKET_ONE_PATH); - QuotaUsageResponse quBucketRes = (QuotaUsageResponse) bucketRes.getEntity(); - assertEquals(BUCKET_ONE_QUOTA, quBucketRes.getQuota()); - assertEquals(BUCKET_ONE_DATA_SIZE, quBucketRes.getQuotaUsed()); - - Response bucketRes2 = nsSummaryEndpoint.getQuotaUsage(BUCKET_TWO_PATH); - QuotaUsageResponse quBucketRes2 = - (QuotaUsageResponse) bucketRes2.getEntity(); - assertEquals(BUCKET_TWO_QUOTA, quBucketRes2.getQuota()); - assertEquals(BUCKET_TWO_DATA_SIZE, quBucketRes2.getQuotaUsed()); - - // other level not applicable - Response naResponse1 = nsSummaryEndpoint.getQuotaUsage(DIR_ONE_PATH); - QuotaUsageResponse quotaUsageResponse1 = - (QuotaUsageResponse) naResponse1.getEntity(); - assertEquals(ResponseStatus.TYPE_NOT_APPLICABLE, - quotaUsageResponse1.getResponseCode()); - - Response naResponse2 = nsSummaryEndpoint.getQuotaUsage(KEY_PATH); - QuotaUsageResponse quotaUsageResponse2 = - (QuotaUsageResponse) naResponse2.getEntity(); - assertEquals(ResponseStatus.TYPE_NOT_APPLICABLE, - quotaUsageResponse2.getResponseCode()); - - // invalid path request - Response invalidRes = nsSummaryEndpoint.getQuotaUsage(INVALID_PATH); - QuotaUsageResponse invalidResObj = - (QuotaUsageResponse) invalidRes.getEntity(); - assertEquals(ResponseStatus.PATH_NOT_FOUND, - invalidResObj.getResponseCode()); - } - - @Test - public void testFileSizeDist() throws Exception { - checkFileSizeDist(ROOT_PATH, 5, 3, 4, 2); - checkFileSizeDist(VOL_PATH, 2, 1, 2, 2); - checkFileSizeDist(BUCKET_ONE_PATH, 1, 1, 1, 2); - checkFileSizeDist(DIR_ONE_PATH, 0, 1, 1, 2); - } - - public void checkFileSizeDist(String path, int bin0, - int bin1, int bin2, int bin3) throws Exception { - Response res = nsSummaryEndpoint.getFileSizeDistribution(path); - FileSizeDistributionResponse fileSizeDistResObj = - (FileSizeDistributionResponse) res.getEntity(); - int[] fileSizeDist = fileSizeDistResObj.getFileSizeDist(); - assertEquals(bin0, fileSizeDist[0]); - assertEquals(bin1, fileSizeDist[1]); - assertEquals(bin2, fileSizeDist[2]); - assertEquals(bin3, fileSizeDist[3]); - for (int i = 4; i < ReconConstants.NUM_OF_FILE_SIZE_BINS; ++i) { - assertEquals(0, fileSizeDist[i]); - } - } - - @Test - public void testConstructFullPath() throws IOException { - OmKeyInfo keyInfo = new OmKeyInfo.Builder() - .setKeyName("file2") - .setVolumeName(VOL) - .setBucketName(BUCKET_ONE) - .setObjectID(KEY_TWO_OBJECT_ID) - .setParentObjectID(DIR_TWO_OBJECT_ID) - .build(); - // Call constructFullPath and verify the result - String fullPath = ReconUtils.constructFullPath(keyInfo, - reconNamespaceSummaryManager); - String expectedPath = "vol/bucket1/dir1/dir2/file2"; - Assertions.assertEquals(expectedPath, fullPath); - - // Create key info for file 3 - keyInfo = new OmKeyInfo.Builder() - .setKeyName("file3") - .setVolumeName(VOL) - .setBucketName(BUCKET_ONE) - .setObjectID(KEY_THREE_OBJECT_ID) - .setParentObjectID(DIR_THREE_OBJECT_ID) - .build(); - fullPath = ReconUtils.constructFullPath(keyInfo, - reconNamespaceSummaryManager); - expectedPath = "vol/bucket1/dir1/dir3/file3"; - Assertions.assertEquals(expectedPath, fullPath); - - // Create key info for file 6 - keyInfo = new OmKeyInfo.Builder() - .setKeyName("file6") - .setVolumeName(VOL) - .setBucketName(BUCKET_ONE) - .setObjectID(KEY_SIX_OBJECT_ID) - .setParentObjectID(DIR_FOUR_OBJECT_ID) - .build(); - fullPath = ReconUtils.constructFullPath(keyInfo, - reconNamespaceSummaryManager); - expectedPath = "vol/bucket1/dir1/dir4/file6"; - Assertions.assertEquals(expectedPath, fullPath); - - // Create key info for file 1 - keyInfo = new OmKeyInfo.Builder() - .setKeyName("file1") - .setVolumeName(VOL) - .setBucketName(BUCKET_ONE) - .setObjectID(KEY_ONE_OBJECT_ID) - .setParentObjectID(BUCKET_ONE_OBJECT_ID) - .build(); - fullPath = ReconUtils.constructFullPath(keyInfo, - reconNamespaceSummaryManager); - expectedPath = "vol/bucket1/file1"; - Assertions.assertEquals(expectedPath, fullPath); - - // Create key info for file 9 - keyInfo = new OmKeyInfo.Builder() - .setKeyName("file9") - .setVolumeName(VOL_TWO) - .setBucketName(BUCKET_THREE) - .setObjectID(KEY_NINE_OBJECT_ID) - .setParentObjectID(DIR_FIVE_OBJECT_ID) - .build(); - fullPath = ReconUtils.constructFullPath(keyInfo, - reconNamespaceSummaryManager); - expectedPath = "vol2/bucket3/dir5/file9"; - Assertions.assertEquals(expectedPath, fullPath); - - // Check for when we encounter a NSSUmamry with parentId -1 - // Fetch NSSummary for dir1 and immediately update its parentId. - NSSummary dir1Summary = reconNamespaceSummaryManager.getNSSummary(DIR_ONE_OBJECT_ID); - dir1Summary.setParentId(-1); // Update parentId to -1 - - reconNamespaceSummaryManager.deleteNSSummary(DIR_ONE_OBJECT_ID); - reconNamespaceSummaryManager.storeNSSummary(DIR_ONE_OBJECT_ID, dir1Summary); - - NSSummary changedDir1Summary = reconNamespaceSummaryManager.getNSSummary(DIR_ONE_OBJECT_ID); - Assertions.assertEquals(-1, changedDir1Summary.getParentId(), "The parentId should be updated to -1"); - - keyInfo = new OmKeyInfo.Builder() - .setKeyName("file2") - .setVolumeName(VOL) - .setBucketName(BUCKET_ONE) - .setObjectID(KEY_TWO_OBJECT_ID) - .setParentObjectID(DIR_TWO_OBJECT_ID) - .build(); - // Call constructFullPath and verify the result - should return empty string when NSSummary parent is invalid - fullPath = ReconUtils.constructFullPath(keyInfo, reconNamespaceSummaryManager); - Assertions.assertEquals("", fullPath, "Should return empty string when NSSummary tree is being rebuilt"); - } - - @Test - public void testConstructFullPathWithNegativeParentIdTriggersRebuild() throws IOException { - // Setup - long dirOneObjectId = 1L; // Sample object ID for the directory - ReconNamespaceSummaryManager mockSummaryManager = mock(ReconNamespaceSummaryManager.class); - NSSummary dir1Summary = new NSSummary(); - dir1Summary.setParentId(-1); // Simulate directory at the top of the tree - when(mockSummaryManager.getNSSummary(dirOneObjectId)).thenReturn(dir1Summary); - - OmKeyInfo keyInfo = new OmKeyInfo.Builder() - .setKeyName("file2") - .setVolumeName("vol") - .setBucketName("bucket1") - .setObjectID(2L) - .setParentObjectID(dirOneObjectId) - .build(); - - // Should return empty string when NSSummary has invalid parentId - String fullPath = ReconUtils.constructFullPath(keyInfo, mockSummaryManager); - Assertions.assertEquals("", fullPath, "Should return empty string when NSSummary has negative parentId"); - } - - @Test - public void testDataSizeUnderVolumeWithRatisReplication()throws IOException { - Response keyResponse = nsSummaryEndpoint.getDiskUsage(VOL_THREE_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(QuotaUtil.getReplicatedSize(replicaDUResponse.getSize(), RatisReplicationConfig.getInstance( - HddsProtos.ReplicationFactor.THREE)), replicaDUResponse.getSizeWithReplica()); - } - - @Test - public void testDataSizeUnderBucketWithRatisReplication()throws IOException { - Response keyResponse = nsSummaryEndpoint.getDiskUsage(VOL_THREE_PATH + "/" + BUCKET_FIVE, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(QuotaUtil.getReplicatedSize(replicaDUResponse.getSize(), RatisReplicationConfig.getInstance( - HddsProtos.ReplicationFactor.THREE)), replicaDUResponse.getSizeWithReplica()); - } - - @Test - public void testDataSizeUnderDirWithRatisReplication()throws IOException { - Response keyResponse = nsSummaryEndpoint.getDiskUsage(VOL_THREE_PATH + "/" + BUCKET_FIVE + "/" + DIR_SIX, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(QuotaUtil.getReplicatedSize(replicaDUResponse.getSize(), RatisReplicationConfig.getInstance( - HddsProtos.ReplicationFactor.THREE)), replicaDUResponse.getSizeWithReplica()); - } - - /** - * Write directories and keys info into OM DB. - * @throws Exception - */ - private void populateOMDB() throws Exception { - // write all directories - writeDirToOm(reconOMMetadataManager, DIR_ONE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, DIR_ONE); - writeDirToOm(reconOMMetadataManager, DIR_TWO_OBJECT_ID, - DIR_ONE_OBJECT_ID, BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, DIR_TWO); - writeDirToOm(reconOMMetadataManager, DIR_THREE_OBJECT_ID, - DIR_ONE_OBJECT_ID, BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, DIR_THREE); - writeDirToOm(reconOMMetadataManager, DIR_FOUR_OBJECT_ID, - DIR_ONE_OBJECT_ID, BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, DIR_FOUR); - writeDirToOm(reconOMMetadataManager, DIR_FIVE_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, DIR_FIVE); - - // write all keys - writeKeyToOm(reconOMMetadataManager, - KEY_ONE, - BUCKET_ONE, - VOL, - FILE_ONE, - KEY_ONE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - KEY_ONE_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_TWO, - BUCKET_ONE, - VOL, - FILE_TWO, - KEY_TWO_OBJECT_ID, - DIR_TWO_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - KEY_TWO_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_THREE, - BUCKET_ONE, - VOL, - FILE_THREE, - KEY_THREE_OBJECT_ID, - DIR_THREE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - KEY_THREE_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_FOUR, - BUCKET_TWO, - VOL, - FILE_FOUR, - KEY_FOUR_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - VOL_OBJECT_ID, - KEY_FOUR_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_FIVE, - BUCKET_TWO, - VOL, - FILE_FIVE, - KEY_FIVE_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - VOL_OBJECT_ID, - KEY_FIVE_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_SIX, - BUCKET_ONE, - VOL, - FILE_SIX, - KEY_SIX_OBJECT_ID, - DIR_FOUR_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - KEY_SIX_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_EIGHT, - BUCKET_THREE, - VOL_TWO, - FILE_EIGHT, - KEY_EIGHT_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - KEY_EIGHT_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_NINE, - BUCKET_THREE, - VOL_TWO, - FILE_NINE, - KEY_NINE_OBJECT_ID, - DIR_FIVE_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - KEY_NINE_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_TEN, - BUCKET_THREE, - VOL_TWO, - FILE_TEN, - KEY_TEN_OBJECT_ID, - DIR_FIVE_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - KEY_TEN_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_ELEVEN, - BUCKET_FOUR, - VOL_TWO, - FILE_ELEVEN, - KEY_ELEVEN_OBJECT_ID, - BUCKET_FOUR_OBJECT_ID, - BUCKET_FOUR_OBJECT_ID, - VOL_TWO_OBJECT_ID, - KEY_ELEVEN_SIZE, - getBucketLayout()); - - } - - private void populateVolumeThree() throws IOException { - - writeDirToOm(reconOMMetadataManager, DIR_SIX_OBJECT_ID, - BUCKET_FIVE_OBJECT_ID, BUCKET_FIVE_OBJECT_ID, - VOL_THREE_OBJECT_ID, DIR_SIX); - - writeDirToOm(reconOMMetadataManager, DIR_SEVEN_OBJECT_ID, - DIR_SIX_OBJECT_ID, BUCKET_FIVE_OBJECT_ID, - VOL_THREE_OBJECT_ID, DIR_SEVEN); - - writeKeyToOm(reconOMMetadataManager, - KEY_TWELVE, - BUCKET_FIVE, - VOL_THREE, - FILE_TWELVE, - FILE_TWELVE_OBJECT_ID, - BUCKET_FIVE_OBJECT_ID, - BUCKET_FIVE_OBJECT_ID, - VOL_THREE_OBJECT_ID, - KEY_TWELVE_SIZE, - getBucketLayout(), RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE), - Time.now(), true); - - writeKeyToOm(reconOMMetadataManager, - KEY_THIRTEEN, - BUCKET_FIVE, - VOL_THREE, - FILE_THIRTEEN, - FILE_THIRTEEN_OBJECT_ID, - DIR_SIX_OBJECT_ID, - BUCKET_FIVE_OBJECT_ID, - VOL_THREE_OBJECT_ID, - KEY_THIRTEEN_SIZE, - getBucketLayout(), RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE), - Time.now(), true); - - writeKeyToOm(reconOMMetadataManager, - KEY_FOURTEEN, - BUCKET_FIVE, - VOL_THREE, - FILE_FOURTEEN, - FILE_FOURTEEN_OBJECT_ID, - DIR_SEVEN_OBJECT_ID, - BUCKET_FIVE_OBJECT_ID, - VOL_THREE_OBJECT_ID, - KEY_FOURTEEN_SIZE, - getBucketLayout(), RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE), - Time.now(), true); - } - - /** - * Create a new OM Metadata manager instance with one user, one vol, and two - * buckets. - * @throws IOException ioEx - */ - private static OMMetadataManager initializeNewOmMetadataManager( - File omDbDir) - throws IOException { - OzoneConfiguration omConfiguration = new OzoneConfiguration(); - omConfiguration.set(OZONE_OM_DB_DIRS, - omDbDir.getAbsolutePath()); - OMMetadataManager omMetadataManager = new OmMetadataManagerImpl( - omConfiguration, null); - - String volumeKey = omMetadataManager.getVolumeKey(VOL); - OmVolumeArgs args = - OmVolumeArgs.newBuilder() - .setObjectID(VOL_OBJECT_ID) - .setVolume(VOL) - .setAdminName(TEST_USER) - .setOwnerName(TEST_USER) - .setQuotaInBytes(VOL_QUOTA) - .build(); - - String volume2Key = omMetadataManager.getVolumeKey(VOL_TWO); - OmVolumeArgs args2 = - OmVolumeArgs.newBuilder() - .setObjectID(VOL_TWO_OBJECT_ID) - .setVolume(VOL_TWO) - .setAdminName(TEST_USER) - .setOwnerName(TEST_USER) - .setQuotaInBytes(VOL_TWO_QUOTA) - .build(); - - String volume3Key = omMetadataManager.getVolumeKey(VOL_THREE); - OmVolumeArgs args3 = - OmVolumeArgs.newBuilder() - .setObjectID(VOL_THREE_OBJECT_ID) - .setVolume(VOL_THREE) - .setAdminName(TEST_USER) - .setOwnerName(TEST_USER) - .setQuotaInBytes(VOL_THREE_QUOTA) - .build(); - - omMetadataManager.getVolumeTable().put(volumeKey, args); - omMetadataManager.getVolumeTable().put(volume2Key, args2); - omMetadataManager.getVolumeTable().put(volume3Key, args3); - - OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() - .setVolumeName(VOL) - .setBucketName(BUCKET_ONE) - .setObjectID(BUCKET_ONE_OBJECT_ID) - .setQuotaInBytes(BUCKET_ONE_QUOTA) - .setBucketLayout(getBucketLayout()) - .build(); - - OmBucketInfo bucketInfo2 = OmBucketInfo.newBuilder() - .setVolumeName(VOL) - .setBucketName(BUCKET_TWO) - .setObjectID(BUCKET_TWO_OBJECT_ID) - .setQuotaInBytes(BUCKET_TWO_QUOTA) - .setBucketLayout(getBucketLayout()) - .build(); - - OmBucketInfo bucketInfo3 = OmBucketInfo.newBuilder() - .setVolumeName(VOL_TWO) - .setBucketName(BUCKET_THREE) - .setObjectID(BUCKET_THREE_OBJECT_ID) - .setQuotaInBytes(BUCKET_THREE_QUOTA) - .setBucketLayout(getBucketLayout()) - .build(); - - OmBucketInfo bucketInfo4 = OmBucketInfo.newBuilder() - .setVolumeName(VOL_TWO) - .setBucketName(BUCKET_FOUR) - .setObjectID(BUCKET_FOUR_OBJECT_ID) - .setQuotaInBytes(BUCKET_FOUR_QUOTA) - .setBucketLayout(getBucketLayout()) - .build(); - - OmBucketInfo bucketInfo5 = OmBucketInfo.newBuilder() - .setVolumeName(VOL_THREE) - .setBucketName(BUCKET_FIVE) - .setObjectID(BUCKET_FIVE_OBJECT_ID) - .setQuotaInBytes(BUCKET_FIVE_QUOTA) - .setBucketLayout(getBucketLayout()) - .build(); - - String bucketKey = omMetadataManager.getBucketKey( - bucketInfo.getVolumeName(), bucketInfo.getBucketName()); - String bucketKey2 = omMetadataManager.getBucketKey( - bucketInfo2.getVolumeName(), bucketInfo2.getBucketName()); - String bucketKey3 = omMetadataManager.getBucketKey( - bucketInfo3.getVolumeName(), bucketInfo3.getBucketName()); - String bucketKey4 = omMetadataManager.getBucketKey( - bucketInfo4.getVolumeName(), bucketInfo4.getBucketName()); - String bucketKey5 = omMetadataManager.getBucketKey( - bucketInfo5.getVolumeName(), bucketInfo5.getBucketName()); - - omMetadataManager.getBucketTable().put(bucketKey, bucketInfo); - omMetadataManager.getBucketTable().put(bucketKey2, bucketInfo2); - omMetadataManager.getBucketTable().put(bucketKey3, bucketInfo3); - omMetadataManager.getBucketTable().put(bucketKey4, bucketInfo4); - omMetadataManager.getBucketTable().put(bucketKey5, bucketInfo5); - - return omMetadataManager; - } - - private void setUpMultiBlockKey() throws IOException { - OmKeyLocationInfoGroup locationInfoGroup = - getLocationInfoGroup1(); - - // add the multi-block key to Recon's OM - writeKeyToOm(reconOMMetadataManager, - MULTI_BLOCK_KEY, - BUCKET_ONE, - VOL, - MULTI_BLOCK_FILE, - MULTI_BLOCK_KEY_OBJECT_ID, - DIR_ONE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup), - getBucketLayout(), - KEY_SEVEN_SIZE); - } - - private OmKeyLocationInfoGroup getLocationInfoGroup1() { - List locationInfoList = new ArrayList<>(); - BlockID block1 = new BlockID(CONTAINER_ONE_ID, 0L); - BlockID block2 = new BlockID(CONTAINER_TWO_ID, 0L); - BlockID block3 = new BlockID(CONTAINER_THREE_ID, 0L); - - OmKeyLocationInfo location1 = new OmKeyLocationInfo.Builder() - .setBlockID(block1) - .setLength(BLOCK_ONE_LENGTH) - .build(); - OmKeyLocationInfo location2 = new OmKeyLocationInfo.Builder() - .setBlockID(block2) - .setLength(BLOCK_TWO_LENGTH) - .build(); - OmKeyLocationInfo location3 = new OmKeyLocationInfo.Builder() - .setBlockID(block3) - .setLength(BLOCK_THREE_LENGTH) - .build(); - locationInfoList.add(location1); - locationInfoList.add(location2); - locationInfoList.add(location3); - - return new OmKeyLocationInfoGroup(0L, locationInfoList); - } - - /** - * Testing the following case. - * vol - * / \ - * bucket1 bucket2 - * / \ / \ - * file1 dir1 file4 file5 - * / \ \ \ - * dir2 dir3 dir4 file7 - * / \ \ - * file2 file3 file6 - * ---------------------------------------- - * vol2 - * / \ - * bucket3 bucket4 - * / \ / - * file8 dir5 file11 - * / \ - * file9 file10 - * Write these keys to OM and - * replicate them. - */ - private OmKeyLocationInfoGroup getLocationInfoGroup2() { - List locationInfoList = new ArrayList<>(); - BlockID block4 = new BlockID(CONTAINER_FOUR_ID, 0L); - BlockID block5 = new BlockID(CONTAINER_FIVE_ID, 0L); - BlockID block6 = new BlockID(CONTAINER_SIX_ID, 0L); - - OmKeyLocationInfo location4 = new OmKeyLocationInfo.Builder() - .setBlockID(block4) - .setLength(BLOCK_FOUR_LENGTH) - .build(); - OmKeyLocationInfo location5 = new OmKeyLocationInfo.Builder() - .setBlockID(block5) - .setLength(BLOCK_FIVE_LENGTH) - .build(); - OmKeyLocationInfo location6 = new OmKeyLocationInfo.Builder() - .setBlockID(block6) - .setLength(BLOCK_SIX_LENGTH) - .build(); - locationInfoList.add(location4); - locationInfoList.add(location5); - locationInfoList.add(location6); - return new OmKeyLocationInfoGroup(0L, locationInfoList); - - } - - @SuppressWarnings("checkstyle:MethodLength") - private void setUpMultiBlockReplicatedKeys() throws IOException { - OmKeyLocationInfoGroup locationInfoGroup1 = - getLocationInfoGroup1(); - OmKeyLocationInfoGroup locationInfoGroup2 = - getLocationInfoGroup2(); - - //vol/bucket1/file1 - writeKeyToOm(reconOMMetadataManager, - KEY_ONE, - BUCKET_ONE, - VOL, - FILE_ONE, - KEY_ONE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getBucketLayout(), - KEY_ONE_SIZE); - - //vol/bucket1/dir1/dir2/file2 - writeKeyToOm(reconOMMetadataManager, - KEY_TWO, - BUCKET_ONE, - VOL, - FILE_TWO, - KEY_TWO_OBJECT_ID, - DIR_TWO_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup2), - getBucketLayout(), - KEY_TWO_SIZE); - - //vol/bucket1/dir1/dir3/file3 - writeKeyToOm(reconOMMetadataManager, - KEY_THREE, - BUCKET_ONE, - VOL, - FILE_THREE, - KEY_THREE_OBJECT_ID, - DIR_THREE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getBucketLayout(), - KEY_THREE_SIZE); - - //vol/bucket2/file4 - writeKeyToOm(reconOMMetadataManager, - KEY_FOUR, - BUCKET_TWO, - VOL, - FILE_FOUR, - KEY_FOUR_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup2), - getBucketLayout(), - KEY_FOUR_SIZE); - - //vol/bucket2/file5 - writeKeyToOm(reconOMMetadataManager, - KEY_FIVE, - BUCKET_TWO, - VOL, - FILE_FIVE, - KEY_FIVE_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getBucketLayout(), - KEY_FIVE_SIZE); - - //vol/bucket1/dir1/dir4/file6 - writeKeyToOm(reconOMMetadataManager, - KEY_SIX, - BUCKET_ONE, - VOL, - FILE_SIX, - KEY_SIX_OBJECT_ID, - DIR_FOUR_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup2), - getBucketLayout(), - KEY_SIX_SIZE); - - //vol/bucket1/dir1/file7 - writeKeyToOm(reconOMMetadataManager, - KEY_SEVEN, - BUCKET_ONE, - VOL, - FILE_SEVEN, - KEY_SEVEN_OBJECT_ID, - DIR_ONE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getBucketLayout(), - KEY_SEVEN_SIZE); - - //vol2/bucket3/file8 - writeKeyToOm(reconOMMetadataManager, - KEY_EIGHT, - BUCKET_THREE, - VOL_TWO, - FILE_EIGHT, - KEY_EIGHT_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - Collections.singletonList(locationInfoGroup2), - getBucketLayout(), - KEY_EIGHT_SIZE); - - //vol2/bucket3/dir5/file9 - writeKeyToOm(reconOMMetadataManager, - KEY_NINE, - BUCKET_THREE, - VOL_TWO, - FILE_NINE, - KEY_NINE_OBJECT_ID, - DIR_FIVE_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getBucketLayout(), - KEY_NINE_SIZE); - - //vol2/bucket3/dir5/file10 - writeKeyToOm(reconOMMetadataManager, - KEY_TEN, - BUCKET_THREE, - VOL_TWO, - FILE_TEN, - KEY_TEN_OBJECT_ID, - DIR_FIVE_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - Collections.singletonList(locationInfoGroup2), - getBucketLayout(), - KEY_TEN_SIZE); - - //vol2/bucket4/file11 - writeKeyToOm(reconOMMetadataManager, - KEY_ELEVEN, - BUCKET_FOUR, - VOL_TWO, - FILE_ELEVEN, - KEY_ELEVEN_OBJECT_ID, - BUCKET_FOUR_OBJECT_ID, - BUCKET_FOUR_OBJECT_ID, - VOL_TWO_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getBucketLayout(), - KEY_ELEVEN_SIZE); - } - - /** - * Generate a set of mock container replica with a size of - * replication factor for container. - * @param replicationFactor number of replica - * @param containerID the container replicated based upon - * @return a set of container replica for testing - */ - private static Set generateMockContainerReplicas( - int replicationFactor, ContainerID containerID) { - Set result = new HashSet<>(); - for (int i = 0; i < replicationFactor; ++i) { - DatanodeDetails randomDatanode = randomDatanodeDetails(); - ContainerReplica replica = new ContainerReplica.ContainerReplicaBuilder() - .setContainerID(containerID) - .setContainerState(State.OPEN) - .setDatanodeDetails(randomDatanode) - .build(); - result.add(replica); - } - return result; - } - - private static ReconStorageContainerManagerFacade getMockReconSCM() - throws ContainerNotFoundException { - ReconStorageContainerManagerFacade reconSCM = - mock(ReconStorageContainerManagerFacade.class); - ContainerManager containerManager = mock(ContainerManager.class); - - // Container 1 is 3-way replicated - ContainerID containerID1 = ContainerID.valueOf(CONTAINER_ONE_ID); - Set containerReplicas1 = generateMockContainerReplicas( - CONTAINER_ONE_REPLICA_COUNT, containerID1); - when(containerManager.getContainerReplicas(containerID1)) - .thenReturn(containerReplicas1); - - // Container 2 is under replicated with 2 replica - ContainerID containerID2 = ContainerID.valueOf(CONTAINER_TWO_ID); - Set containerReplicas2 = generateMockContainerReplicas( - CONTAINER_TWO_REPLICA_COUNT, containerID2); - when(containerManager.getContainerReplicas(containerID2)) - .thenReturn(containerReplicas2); - - // Container 3 is over replicated with 4 replica - ContainerID containerID3 = ContainerID.valueOf(CONTAINER_THREE_ID); - Set containerReplicas3 = generateMockContainerReplicas( - CONTAINER_THREE_REPLICA_COUNT, containerID3); - when(containerManager.getContainerReplicas(containerID3)) - .thenReturn(containerReplicas3); - - // Container 4 is replicated with 5 replica - ContainerID containerID4 = ContainerID.valueOf(CONTAINER_FOUR_ID); - Set containerReplicas4 = generateMockContainerReplicas( - CONTAINER_FOUR_REPLICA_COUNT, containerID4); - when(containerManager.getContainerReplicas(containerID4)) - .thenReturn(containerReplicas4); - - // Container 5 is replicated with 2 replica - ContainerID containerID5 = ContainerID.valueOf(CONTAINER_FIVE_ID); - Set containerReplicas5 = generateMockContainerReplicas( - CONTAINER_FIVE_REPLICA_COUNT, containerID5); - when(containerManager.getContainerReplicas(containerID5)) - .thenReturn(containerReplicas5); - - // Container 6 is replicated with 3 replica - ContainerID containerID6 = ContainerID.valueOf(CONTAINER_SIX_ID); - Set containerReplicas6 = generateMockContainerReplicas( - CONTAINER_SIX_REPLICA_COUNT, containerID6); - when(containerManager.getContainerReplicas(containerID6)) - .thenReturn(containerReplicas6); - - when(reconSCM.getContainerManager()).thenReturn(containerManager); - ReconNodeManager mockReconNodeManager = mock(ReconNodeManager.class); - when(mockReconNodeManager.getStats()).thenReturn(getMockSCMRootStat()); - when(reconSCM.getScmNodeManager()).thenReturn(mockReconNodeManager); - return reconSCM; - } - - private static BucketLayout getBucketLayout() { - return BucketLayout.FILE_SYSTEM_OPTIMIZED; - } - - private static SCMNodeStat getMockSCMRootStat() { - return new SCMNodeStat(ROOT_QUOTA, ROOT_DATA_SIZE, - ROOT_QUOTA - ROOT_DATA_SIZE, 0, ROOT_QUOTA - ROOT_DATA_SIZE - 1, 0); - } -} diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java deleted file mode 100644 index 7ac14a9eb4d1..000000000000 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithLegacy.java +++ /dev/null @@ -1,1491 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.recon.api; - -import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE; -import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS; -import static org.apache.hadoop.ozone.om.helpers.QuotaUtil.getReplicatedSize; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getMockOzoneManagerServiceProvider; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.setConfiguration; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDirToOm; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import javax.ws.rs.core.Response; -import org.apache.hadoop.hdds.client.BlockID; -import org.apache.hadoop.hdds.client.RatisReplicationConfig; -import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State; -import org.apache.hadoop.hdds.scm.container.ContainerID; -import org.apache.hadoop.hdds.scm.container.ContainerManager; -import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; -import org.apache.hadoop.hdds.scm.container.ContainerReplica; -import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat; -import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.om.OMConfigKeys; -import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; -import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; -import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; -import org.apache.hadoop.ozone.om.helpers.QuotaUtil; -import org.apache.hadoop.ozone.recon.ReconConstants; -import org.apache.hadoop.ozone.recon.ReconTestInjector; -import org.apache.hadoop.ozone.recon.ReconUtils; -import org.apache.hadoop.ozone.recon.api.handlers.BucketHandler; -import org.apache.hadoop.ozone.recon.api.handlers.EntityHandler; -import org.apache.hadoop.ozone.recon.api.types.DUResponse; -import org.apache.hadoop.ozone.recon.api.types.FileSizeDistributionResponse; -import org.apache.hadoop.ozone.recon.api.types.QuotaUsageResponse; -import org.apache.hadoop.ozone.recon.api.types.ResponseStatus; -import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; -import org.apache.hadoop.ozone.recon.scm.ReconNodeManager; -import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade; -import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; -import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; -import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; -import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; -import org.apache.hadoop.ozone.recon.tasks.NSSummaryTaskWithLegacy; -import org.apache.hadoop.util.Time; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -/** - * Test for NSSummary REST APIs with Legacy. - * We tested on a mini file system with the following setting: - * vol - * / \ - * bucket1 bucket2 - * / \ / \ - * file1 dir1 file4 file5 - * / \ \ - * dir2 dir3 dir4 - * / \ \ - * file2 file3 file6 - * ---------------------------------------- - * vol2 - * / \ - * bucket3 bucket4 - * / \ / - * file8 dir5 file11 - * / \ - * file9 file10 - * ---------------------------------------- - * vol3 - * | - * bucket5 - * / \ - * file12 dir6 - * / \ - * file13 dir7 - * / - * file14 - * This is a test for the Rest APIs only. We have tested NSSummaryTask before, - * so there is no need to test process() on DB's updates - */ -public class TestNSSummaryEndpointWithLegacy extends NSSummaryTests { - @TempDir - private Path temporaryFolder; - - private ReconNamespaceSummaryManager reconNamespaceSummaryManager; - private ReconOMMetadataManager reconOMMetadataManager; - private NSSummaryEndpoint nsSummaryEndpoint; - - private static final String TEST_PATH_UTILITY = - "/vol1/buck1/a/b/c/d/e/file1.txt"; - private static final String PARENT_DIR = "vol1/buck1/a/b/c/d/e"; - private static final String[] TEST_NAMES = - new String[]{"vol1", "buck1", "a", "b", "c", "d", "e", "file1.txt"}; - private static final String TEST_KEY_NAMES = "a/b/c/d/e/file1.txt"; - - // Object names - private static final String VOL = "vol"; - private static final String VOL_TWO = "vol2"; - private static final String VOL_THREE = "vol3"; - private static final String BUCKET_ONE = "bucket1"; - private static final String BUCKET_TWO = "bucket2"; - private static final String BUCKET_THREE = "bucket3"; - private static final String BUCKET_FOUR = "bucket4"; - private static final String BUCKET_FIVE = "bucket5"; - private static final String KEY_ONE = "file1"; - private static final String KEY_TWO = "dir1/dir2/file2"; - private static final String KEY_THREE = "dir1/dir3/file3"; - private static final String KEY_FOUR = "file4"; - private static final String KEY_FIVE = "file5"; - private static final String KEY_SIX = "dir1/dir4/file6"; - private static final String KEY_SEVEN = "dir1/file7"; - private static final String KEY_EIGHT = "file8"; - private static final String KEY_NINE = "dir5/file9"; - private static final String KEY_TEN = "dir5/file10"; - private static final String KEY_ELEVEN = "file11"; - private static final String MULTI_BLOCK_KEY = "dir1/file7"; - private static final String MULTI_BLOCK_FILE = "file7"; - private static final String KEY_TWELVE = "file12"; - private static final String KEY_THIRTEEN = "dir6/file13"; - private static final String KEY_FOURTEEN = "dir6/dir7/file14"; - - private static final String FILE_ONE = "file1"; - private static final String FILE_TWO = "file2"; - private static final String FILE_THREE = "file3"; - private static final String FILE_FOUR = "file4"; - private static final String FILE_FIVE = "file5"; - private static final String FILE_SIX = "file6"; - private static final String FILE_SEVEN = "file7"; - private static final String FILE_EIGHT = "file8"; - private static final String FILE_NINE = "file9"; - private static final String FILE_TEN = "file10"; - private static final String FILE_ELEVEN = "file11"; - private static final String FILE_TWELVE = "file12"; - private static final String FILE_THIRTEEN = "file13"; - private static final String FILE_FOURTEEN = "file14"; - - private static final String DIR_ONE = "dir1"; - private static final String DIR_TWO = "dir2"; - private static final String DIR_THREE = "dir3"; - private static final String DIR_FOUR = "dir4"; - private static final String DIR_FIVE = "dir5"; - private static final String DIR_SIX = "dir6"; - private static final String DIR_SEVEN = "dir7"; - // objects IDs - private static final long PARENT_OBJECT_ID_ZERO = 0L; - private static final long VOL_OBJECT_ID = 0L; - private static final long BUCKET_ONE_OBJECT_ID = 1L; - private static final long BUCKET_TWO_OBJECT_ID = 2L; - private static final long KEY_ONE_OBJECT_ID = 3L; - private static final long DIR_ONE_OBJECT_ID = 4L; - private static final long KEY_TWO_OBJECT_ID = 5L; - private static final long KEY_FOUR_OBJECT_ID = 6L; - private static final long DIR_TWO_OBJECT_ID = 7L; - private static final long KEY_THREE_OBJECT_ID = 8L; - private static final long KEY_FIVE_OBJECT_ID = 9L; - private static final long KEY_SIX_OBJECT_ID = 10L; - private static final long DIR_THREE_OBJECT_ID = 11L; - private static final long DIR_FOUR_OBJECT_ID = 12L; - private static final long MULTI_BLOCK_KEY_OBJECT_ID = 13L; - private static final long KEY_SEVEN_OBJECT_ID = 13L; - private static final long VOL_TWO_OBJECT_ID = 14L; - private static final long BUCKET_THREE_OBJECT_ID = 15L; - private static final long BUCKET_FOUR_OBJECT_ID = 16L; - private static final long KEY_EIGHT_OBJECT_ID = 17L; - private static final long DIR_FIVE_OBJECT_ID = 18L; - private static final long KEY_NINE_OBJECT_ID = 19L; - private static final long KEY_TEN_OBJECT_ID = 20L; - private static final long KEY_ELEVEN_OBJECT_ID = 21L; - private static final long VOL_THREE_OBJECT_ID = 22L; - private static final long DIR_SIX_OBJECT_ID = 23L; - private static final long DIR_SEVEN_OBJECT_ID = 24L; - private static final long FILE_TWELVE_OBJECT_ID = 25L; - private static final long FILE_THIRTEEN_OBJECT_ID = 26L; - private static final long FILE_FOURTEEN_OBJECT_ID = 27L; - private static final long BUCKET_FIVE_OBJECT_ID = 28L; - - // container IDs - private static final long CONTAINER_ONE_ID = 1L; - private static final long CONTAINER_TWO_ID = 2L; - private static final long CONTAINER_THREE_ID = 3L; - private static final long CONTAINER_FOUR_ID = 4L; - private static final long CONTAINER_FIVE_ID = 5L; - private static final long CONTAINER_SIX_ID = 6L; - - // replication factors - private static final int CONTAINER_ONE_REPLICA_COUNT = 3; - private static final int CONTAINER_TWO_REPLICA_COUNT = 2; - private static final int CONTAINER_THREE_REPLICA_COUNT = 4; - private static final int CONTAINER_FOUR_REPLICA_COUNT = 5; - private static final int CONTAINER_FIVE_REPLICA_COUNT = 2; - private static final int CONTAINER_SIX_REPLICA_COUNT = 3; - - // block lengths - private static final long BLOCK_ONE_LENGTH = 1000L; - private static final long BLOCK_TWO_LENGTH = 2000L; - private static final long BLOCK_THREE_LENGTH = 3000L; - private static final long BLOCK_FOUR_LENGTH = 4000L; - private static final long BLOCK_FIVE_LENGTH = 5000L; - private static final long BLOCK_SIX_LENGTH = 6000L; - - // data size in bytes - private static final long KEY_ONE_SIZE = 500L; // bin 0 - private static final long KEY_TWO_SIZE = OzoneConsts.KB + 1; // bin 1 - private static final long KEY_THREE_SIZE = 4 * OzoneConsts.KB + 1; // bin 3 - private static final long KEY_FOUR_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 - private static final long KEY_FIVE_SIZE = 100L; // bin 0 - private static final long KEY_SIX_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 - private static final long KEY_SEVEN_SIZE = 4 * OzoneConsts.KB + 1; - private static final long KEY_EIGHT_SIZE = OzoneConsts.KB + 1; // bin 1 - private static final long KEY_NINE_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 - private static final long KEY_TEN_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 - private static final long KEY_ELEVEN_SIZE = OzoneConsts.KB + 1; // bin 1 - private static final long KEY_TWELVE_SIZE = OzoneConsts.KB; - private static final long KEY_THIRTEEN_SIZE = OzoneConsts.KB; - private static final long KEY_FOURTEEN_SIZE = OzoneConsts.KB; - - private static final long FILE1_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_ONE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE2_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_TWO_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE3_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_THREE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE4_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_FOUR_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE5_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_FIVE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE6_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_SIX_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE7_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_SEVEN_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE8_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_EIGHT_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE9_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_NINE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE10_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_TEN_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE11_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_ELEVEN_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE12_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_TWELVE_SIZE, RatisReplicationConfig.getInstance(THREE)); - private static final long FILE13_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_THIRTEEN_SIZE, RatisReplicationConfig.getInstance(THREE)); - private static final long FILE14_SIZE_WITH_REPLICA = - getReplicatedSize(KEY_FOURTEEN_SIZE, RatisReplicationConfig.getInstance(THREE)); - - private static final long MULTI_BLOCK_KEY_SIZE_WITH_REPLICA - = FILE7_SIZE_WITH_REPLICA; - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_ROOT - = FILE1_SIZE_WITH_REPLICA - + FILE2_SIZE_WITH_REPLICA - + FILE3_SIZE_WITH_REPLICA - + FILE4_SIZE_WITH_REPLICA - + FILE5_SIZE_WITH_REPLICA - + FILE6_SIZE_WITH_REPLICA - + FILE7_SIZE_WITH_REPLICA - + FILE8_SIZE_WITH_REPLICA - + FILE9_SIZE_WITH_REPLICA - + FILE10_SIZE_WITH_REPLICA - + FILE11_SIZE_WITH_REPLICA - + FILE12_SIZE_WITH_REPLICA - + FILE13_SIZE_WITH_REPLICA - + FILE14_SIZE_WITH_REPLICA; - - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_VOL - = FILE1_SIZE_WITH_REPLICA - + FILE2_SIZE_WITH_REPLICA - + FILE3_SIZE_WITH_REPLICA - + FILE4_SIZE_WITH_REPLICA - + FILE5_SIZE_WITH_REPLICA - + FILE6_SIZE_WITH_REPLICA - + FILE7_SIZE_WITH_REPLICA; - - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET1 - = FILE1_SIZE_WITH_REPLICA - + FILE2_SIZE_WITH_REPLICA - + FILE3_SIZE_WITH_REPLICA - + FILE6_SIZE_WITH_REPLICA - + FILE7_SIZE_WITH_REPLICA; - - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_DIR1 - = FILE2_SIZE_WITH_REPLICA - + FILE3_SIZE_WITH_REPLICA - + FILE6_SIZE_WITH_REPLICA - + FILE7_SIZE_WITH_REPLICA; - - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_DIR2 - = FILE2_SIZE_WITH_REPLICA; - - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_KEY - = FILE4_SIZE_WITH_REPLICA; - - // quota in bytes - private static final long ROOT_QUOTA = 2 * (2 * OzoneConsts.MB); - private static final long VOL_QUOTA = 2 * OzoneConsts.MB; - private static final long VOL_TWO_QUOTA = 2 * OzoneConsts.MB; - private static final long VOL_THREE_QUOTA = 2 * OzoneConsts.MB; - private static final long BUCKET_ONE_QUOTA = OzoneConsts.MB; - private static final long BUCKET_TWO_QUOTA = OzoneConsts.MB; - private static final long BUCKET_THREE_QUOTA = OzoneConsts.MB; - private static final long BUCKET_FOUR_QUOTA = OzoneConsts.MB; - private static final long BUCKET_FIVE_QUOTA = OzoneConsts.MB; - - // mock client's path requests - private static final String TEST_USER = "TestUser"; - private static final String ROOT_PATH = "/"; - private static final String VOL_PATH = "/vol"; - private static final String VOL_TWO_PATH = "/vol2"; - private static final String BUCKET_ONE_PATH = "/vol/bucket1"; - private static final String BUCKET_TWO_PATH = "/vol/bucket2"; - private static final String DIR_ONE_PATH = "/vol/bucket1/dir1"; - private static final String DIR_TWO_PATH = "/vol/bucket1/dir1/dir2"; - private static final String DIR_THREE_PATH = "/vol/bucket1/dir1/dir3"; - private static final String DIR_FOUR_PATH = "/vol/bucket1/dir1/dir4"; - private static final String KEY_PATH = "/vol/bucket2/file4"; - private static final String MULTI_BLOCK_KEY_PATH = "/vol/bucket1/dir1/file7"; - private static final String INVALID_PATH = "/vol/path/not/found"; - private static final String VOL_THREE_PATH = "/vol3"; - - // some expected answers - private static final long ROOT_DATA_SIZE = KEY_ONE_SIZE + KEY_TWO_SIZE + - KEY_THREE_SIZE + KEY_FOUR_SIZE + KEY_FIVE_SIZE + KEY_SIX_SIZE + KEY_SEVEN_SIZE + - KEY_EIGHT_SIZE + KEY_NINE_SIZE + KEY_TEN_SIZE + KEY_ELEVEN_SIZE + - FILE12_SIZE_WITH_REPLICA + FILE13_SIZE_WITH_REPLICA + FILE14_SIZE_WITH_REPLICA; - private static final long VOL_DATA_SIZE = KEY_ONE_SIZE + KEY_TWO_SIZE + - KEY_THREE_SIZE + KEY_FOUR_SIZE + KEY_FIVE_SIZE + KEY_SIX_SIZE + KEY_SEVEN_SIZE; - - private static final long VOL_TWO_DATA_SIZE = - KEY_EIGHT_SIZE + KEY_NINE_SIZE + KEY_TEN_SIZE + KEY_ELEVEN_SIZE; - - private static final long BUCKET_ONE_DATA_SIZE = KEY_ONE_SIZE + KEY_TWO_SIZE + - KEY_THREE_SIZE + KEY_SIX_SIZE + KEY_SEVEN_SIZE; - - private static final long BUCKET_TWO_DATA_SIZE = - KEY_FOUR_SIZE + KEY_FIVE_SIZE; - - private static final long DIR_ONE_DATA_SIZE = KEY_TWO_SIZE + - KEY_THREE_SIZE + KEY_SIX_SIZE + KEY_SEVEN_SIZE; - - @BeforeEach - public void setUp() throws Exception { - OzoneConfiguration conf = new OzoneConfiguration(); - OMMetadataManager omMetadataManager = initializeNewOmMetadataManager( - Files.createDirectory(temporaryFolder.resolve( - "JunitOmDBDir")).toFile(), conf); - OzoneManagerServiceProviderImpl ozoneManagerServiceProvider = - getMockOzoneManagerServiceProvider(); - reconOMMetadataManager = getTestReconOmMetadataManager(omMetadataManager, - Files.createDirectory(temporaryFolder.resolve( - "omMetadatDir")).toFile()); - - ReconTestInjector reconTestInjector = - new ReconTestInjector.Builder(temporaryFolder.toFile()) - .withReconOm(reconOMMetadataManager) - .withOmServiceProvider(ozoneManagerServiceProvider) - .withReconSqlDb() - .withContainerDB() - .addBinding(OzoneStorageContainerManager.class, - getMockReconSCM()) - .addBinding(StorageContainerServiceProvider.class, - mock(StorageContainerServiceProviderImpl.class)) - .addBinding(NSSummaryEndpoint.class) - .build(); - this.reconNamespaceSummaryManager = - reconTestInjector.getInstance(ReconNamespaceSummaryManager.class); - nsSummaryEndpoint = reconTestInjector.getInstance(NSSummaryEndpoint.class); - - // populate OM DB and reprocess into Recon RocksDB - populateOMDB(); - populateVolumeThree(); - setUpMultiBlockReplicatedKeys(); - NSSummaryTaskWithLegacy nsSummaryTaskWithLegacy = - new NSSummaryTaskWithLegacy(reconNamespaceSummaryManager, - reconOMMetadataManager, conf, 10); - nsSummaryTaskWithLegacy.reprocessWithLegacy(reconOMMetadataManager); - } - - @Test - public void testUtility() { - String[] names = EntityHandler.parseRequestPath(TEST_PATH_UTILITY); - assertArrayEquals(TEST_NAMES, names); - String keyName = BucketHandler.getKeyName(names); - assertEquals(TEST_KEY_NAMES, keyName); - String subpath = BucketHandler.buildSubpath(PARENT_DIR, "file1.txt"); - assertEquals(TEST_PATH_UTILITY, subpath); - } - - @Test - public void testGetBasicInfoRoot() throws Exception { - testNSSummaryBasicInfoRoot( - nsSummaryEndpoint, reconOMMetadataManager); - } - - @Test - public void testGetBasicInfoVol() throws Exception { - // Test volume basics - testNSSummaryBasicInfoVolume(nsSummaryEndpoint); - } - - @Test - public void testGetBasicInfoBucketOne() throws Exception { - // Test bucket 1's basics - testNSSummaryBasicInfoBucketOne( - BucketLayout.LEGACY, - nsSummaryEndpoint); - } - - @Test - public void testGetBasicInfoBucketTwo() throws Exception { - // Test bucket 2's basics - testNSSummaryBasicInfoBucketTwo( - BucketLayout.LEGACY, - nsSummaryEndpoint); - } - - @Test - public void testGetBasicInfoDir() throws Exception { - // Test intermediate directory basics - testNSSummaryBasicInfoDir(nsSummaryEndpoint); - } - - @Test - public void testGetBasicInfoNoPath() throws Exception { - // Test invalid path - testNSSummaryBasicInfoNoPath(nsSummaryEndpoint); - } - - @Test - public void testGetBasicInfoKey() throws Exception { - // Test key - testNSSummaryBasicInfoKey(nsSummaryEndpoint); - } - - @Test - public void testDiskUsageRoot() throws Exception { - // root level DU - Response rootResponse = nsSummaryEndpoint.getDiskUsage(ROOT_PATH, - false, false, false); - DUResponse duRootRes = (DUResponse) rootResponse.getEntity(); - assertEquals(3, duRootRes.getCount()); - List duRootData = duRootRes.getDuData(); - // sort based on subpath - Collections.sort(duRootData, - Comparator.comparing(DUResponse.DiskUsage::getSubpath)); - DUResponse.DiskUsage duVol1 = duRootData.get(0); - DUResponse.DiskUsage duVol2 = duRootData.get(1); - assertEquals(VOL_PATH, duVol1.getSubpath()); - assertEquals(VOL_TWO_PATH, duVol2.getSubpath()); - assertEquals(VOL_DATA_SIZE, duVol1.getSize()); - assertEquals(VOL_TWO_DATA_SIZE, duVol2.getSize()); - } - - @Test - public void testDiskUsageVolume() throws Exception { - // volume level DU - Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_PATH, - false, false, false); - DUResponse duVolRes = (DUResponse) volResponse.getEntity(); - assertEquals(2, duVolRes.getCount()); - List duData = duVolRes.getDuData(); - // sort based on subpath - Collections.sort(duData, - Comparator.comparing(DUResponse.DiskUsage::getSubpath)); - DUResponse.DiskUsage duBucket1 = duData.get(0); - DUResponse.DiskUsage duBucket2 = duData.get(1); - assertEquals(BUCKET_ONE_PATH, duBucket1.getSubpath()); - assertEquals(BUCKET_TWO_PATH, duBucket2.getSubpath()); - assertEquals(BUCKET_ONE_DATA_SIZE, duBucket1.getSize()); - assertEquals(BUCKET_TWO_DATA_SIZE, duBucket2.getSize()); - } - - @Test - public void testDiskUsageBucket() throws Exception { - // bucket level DU - Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_ONE_PATH, - false, false, false); - DUResponse duBucketResponse = (DUResponse) bucketResponse.getEntity(); - assertEquals(1, duBucketResponse.getCount()); - DUResponse.DiskUsage duDir1 = duBucketResponse.getDuData().get(0); - assertEquals(DIR_ONE_PATH, duDir1.getSubpath()); - assertEquals(DIR_ONE_DATA_SIZE, duDir1.getSize()); - } - - @Test - public void testDiskUsageDir() throws Exception { - // dir level DU - Response dirResponse = nsSummaryEndpoint.getDiskUsage(DIR_ONE_PATH, - false, false, false); - DUResponse duDirReponse = (DUResponse) dirResponse.getEntity(); - assertEquals(3, duDirReponse.getCount()); - List duSubDir = duDirReponse.getDuData(); - Collections.sort(duSubDir, - Comparator.comparing(DUResponse.DiskUsage::getSubpath)); - DUResponse.DiskUsage duDir2 = duSubDir.get(0); - DUResponse.DiskUsage duDir3 = duSubDir.get(1); - DUResponse.DiskUsage duDir4 = duSubDir.get(2); - assertEquals(DIR_TWO_PATH, duDir2.getSubpath()); - assertEquals(KEY_TWO_SIZE, duDir2.getSize()); - - assertEquals(DIR_THREE_PATH, duDir3.getSubpath()); - assertEquals(KEY_THREE_SIZE, duDir3.getSize()); - - assertEquals(DIR_FOUR_PATH, duDir4.getSubpath()); - assertEquals(KEY_SIX_SIZE, duDir4.getSize()); - } - - @Test - public void testDiskUsageKey() throws Exception { - // key level DU - Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_PATH, - false, false, false); - DUResponse keyObj = (DUResponse) keyResponse.getEntity(); - assertEquals(0, keyObj.getCount()); - assertEquals(KEY_FOUR_SIZE, keyObj.getSize()); - } - - @Test - public void testDiskUsageUnknown() throws Exception { - // invalid path check - Response invalidResponse = nsSummaryEndpoint.getDiskUsage(INVALID_PATH, - false, false, false); - DUResponse invalidObj = (DUResponse) invalidResponse.getEntity(); - assertEquals(ResponseStatus.PATH_NOT_FOUND, - invalidObj.getStatus()); - } - - @Test - public void testDiskUsageWithReplication() throws Exception { - setUpMultiBlockKey(); - Response keyResponse = nsSummaryEndpoint.getDiskUsage(MULTI_BLOCK_KEY_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_KEY_SIZE_WITH_REPLICA, - replicaDUResponse.getSizeWithReplica()); - } - - @Test - public void testDataSizeUnderRootWithReplication() throws IOException { - // withReplica is true - Response rootResponse = nsSummaryEndpoint.getDiskUsage(ROOT_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) rootResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_ROOT, - replicaDUResponse.getSizeWithReplica()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_VOL, - replicaDUResponse.getDuData().get(0).getSizeWithReplica()); - - } - - @Test - public void testDataSizeUnderVolWithReplication() throws IOException { - Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) volResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_VOL, - replicaDUResponse.getSizeWithReplica()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET1, - replicaDUResponse.getDuData().get(0).getSizeWithReplica()); - } - - @Test - public void testDataSizeUnderBucketWithReplication() throws IOException { - Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_ONE_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) bucketResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET1, - replicaDUResponse.getSizeWithReplica()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_DIR1, - replicaDUResponse.getDuData().get(0).getSizeWithReplica()); - } - - /** - * When calculating DU under dir1 - * there are 3 keys, file2, file3, file6. - * There is one direct key, file7. - * @throws IOException - */ - @Test - public void testDataSizeUnderDirWithReplication() throws IOException { - Response dir1Response = nsSummaryEndpoint.getDiskUsage(DIR_ONE_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) dir1Response.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_DIR1, - replicaDUResponse.getSizeWithReplica()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_DIR2, - replicaDUResponse.getDuData().get(0).getSizeWithReplica()); - } - - @Test - public void testDataSizeUnderKeyWithReplication() throws IOException { - Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_KEY, - replicaDUResponse.getSizeWithReplica()); - } - - @Test - public void testQuotaUsage() throws Exception { - // root level quota usage - Response rootResponse = nsSummaryEndpoint.getQuotaUsage(ROOT_PATH); - QuotaUsageResponse quRootRes = - (QuotaUsageResponse) rootResponse.getEntity(); - assertEquals(ROOT_QUOTA, quRootRes.getQuota()); - assertEquals(ROOT_DATA_SIZE, quRootRes.getQuotaUsed()); - - // volume level quota usage - Response volResponse = nsSummaryEndpoint.getQuotaUsage(VOL_PATH); - QuotaUsageResponse quVolRes = (QuotaUsageResponse) volResponse.getEntity(); - assertEquals(VOL_QUOTA, quVolRes.getQuota()); - assertEquals(VOL_DATA_SIZE, quVolRes.getQuotaUsed()); - - // bucket level quota usage - Response bucketRes = nsSummaryEndpoint.getQuotaUsage(BUCKET_ONE_PATH); - QuotaUsageResponse quBucketRes = (QuotaUsageResponse) bucketRes.getEntity(); - assertEquals(BUCKET_ONE_QUOTA, quBucketRes.getQuota()); - assertEquals(BUCKET_ONE_DATA_SIZE, quBucketRes.getQuotaUsed()); - - Response bucketRes2 = nsSummaryEndpoint.getQuotaUsage(BUCKET_TWO_PATH); - QuotaUsageResponse quBucketRes2 = - (QuotaUsageResponse) bucketRes2.getEntity(); - assertEquals(BUCKET_TWO_QUOTA, quBucketRes2.getQuota()); - assertEquals(BUCKET_TWO_DATA_SIZE, quBucketRes2.getQuotaUsed()); - - // other level not applicable - Response naResponse1 = nsSummaryEndpoint.getQuotaUsage(DIR_ONE_PATH); - QuotaUsageResponse quotaUsageResponse1 = - (QuotaUsageResponse) naResponse1.getEntity(); - assertEquals(ResponseStatus.TYPE_NOT_APPLICABLE, - quotaUsageResponse1.getResponseCode()); - - Response naResponse2 = nsSummaryEndpoint.getQuotaUsage(KEY_PATH); - QuotaUsageResponse quotaUsageResponse2 = - (QuotaUsageResponse) naResponse2.getEntity(); - assertEquals(ResponseStatus.TYPE_NOT_APPLICABLE, - quotaUsageResponse2.getResponseCode()); - - // invalid path request - Response invalidRes = nsSummaryEndpoint.getQuotaUsage(INVALID_PATH); - QuotaUsageResponse invalidResObj = - (QuotaUsageResponse) invalidRes.getEntity(); - assertEquals(ResponseStatus.PATH_NOT_FOUND, - invalidResObj.getResponseCode()); - } - - @Test - public void testFileSizeDist() throws Exception { - checkFileSizeDist(ROOT_PATH, 5, 3, 4, 2); - checkFileSizeDist(VOL_PATH, 2, 1, 2, 2); - checkFileSizeDist(BUCKET_ONE_PATH, 1, 1, 1, 2); - checkFileSizeDist(DIR_ONE_PATH, 0, 1, 1, 2); - } - - public void checkFileSizeDist(String path, int bin0, - int bin1, int bin2, int bin3) throws Exception { - Response res = nsSummaryEndpoint.getFileSizeDistribution(path); - FileSizeDistributionResponse fileSizeDistResObj = - (FileSizeDistributionResponse) res.getEntity(); - int[] fileSizeDist = fileSizeDistResObj.getFileSizeDist(); - assertEquals(bin0, fileSizeDist[0]); - assertEquals(bin1, fileSizeDist[1]); - assertEquals(bin2, fileSizeDist[2]); - assertEquals(bin3, fileSizeDist[3]); - for (int i = 4; i < ReconConstants.NUM_OF_FILE_SIZE_BINS; ++i) { - assertEquals(0, fileSizeDist[i]); - } - } - - @Test - public void testConstructFullPath() throws IOException { - // For Key Tables the parent object ID is not set hence it - // will by default be set as -1 when the NSSummary object is created - OmKeyInfo keyInfo = new OmKeyInfo.Builder() - .setKeyName("dir1/dir2/file2") - .setVolumeName(VOL) - .setBucketName(BUCKET_ONE) - .setObjectID(KEY_TWO_OBJECT_ID) - .build(); - // Call constructFullPath and verify the result - String fullPath = ReconUtils.constructFullPath(keyInfo, - reconNamespaceSummaryManager); - String expectedPath = "vol/bucket1/dir1/dir2/file2"; - Assertions.assertEquals(expectedPath, fullPath); - - // Create key info for file 3 - keyInfo = new OmKeyInfo.Builder() - .setKeyName("dir1/dir2/") - .setVolumeName(VOL) - .setBucketName(BUCKET_ONE) - .setObjectID(DIR_TWO_OBJECT_ID) - .build(); - fullPath = ReconUtils.constructFullPath(keyInfo, - reconNamespaceSummaryManager); - expectedPath = "vol/bucket1/dir1/dir2/"; - Assertions.assertEquals(expectedPath, fullPath); - - // Create key info for file 6 - keyInfo = new OmKeyInfo.Builder() - .setKeyName("dir1/dir4/file6") - .setVolumeName(VOL) - .setBucketName(BUCKET_ONE) - .setObjectID(KEY_SIX_OBJECT_ID) - .build(); - fullPath = ReconUtils.constructFullPath(keyInfo, - reconNamespaceSummaryManager); - expectedPath = "vol/bucket1/dir1/dir4/file6"; - Assertions.assertEquals(expectedPath, fullPath); - } - - @Test - public void testDataSizeUnderVolumeWithRatisReplication()throws IOException { - Response keyResponse = nsSummaryEndpoint.getDiskUsage(VOL_THREE_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(QuotaUtil.getReplicatedSize(replicaDUResponse.getSize(), RatisReplicationConfig.getInstance( - HddsProtos.ReplicationFactor.THREE)), replicaDUResponse.getSizeWithReplica()); - } - - @Test - public void testDataSizeUnderBucketWithRatisReplication()throws IOException { - Response keyResponse = nsSummaryEndpoint.getDiskUsage(VOL_THREE_PATH + "/" + BUCKET_FIVE, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(QuotaUtil.getReplicatedSize(replicaDUResponse.getSize(), RatisReplicationConfig.getInstance( - HddsProtos.ReplicationFactor.THREE)), replicaDUResponse.getSizeWithReplica()); - } - - @Test - public void testDataSizeUnderDirWithRatisReplication()throws IOException { - Response keyResponse = nsSummaryEndpoint.getDiskUsage(VOL_THREE_PATH + "/" + BUCKET_FIVE + "/" + DIR_SIX, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(QuotaUtil.getReplicatedSize(replicaDUResponse.getSize(), RatisReplicationConfig.getInstance( - HddsProtos.ReplicationFactor.THREE)), replicaDUResponse.getSizeWithReplica()); - } - - /** - * Write directories and keys info into OM DB. - * @throws Exception - */ - @SuppressWarnings("checkstyle:MethodLength") - private void populateOMDB() throws Exception { - // write all directories - writeDirToOm(reconOMMetadataManager, - (DIR_ONE + OM_KEY_PREFIX), - BUCKET_ONE, - VOL, - DIR_ONE, - DIR_ONE_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - getBucketLayout()); - writeDirToOm(reconOMMetadataManager, - (DIR_ONE + OM_KEY_PREFIX + DIR_TWO + OM_KEY_PREFIX), - BUCKET_ONE, - VOL, - DIR_TWO, - DIR_TWO_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - getBucketLayout()); - writeDirToOm(reconOMMetadataManager, - (DIR_ONE + OM_KEY_PREFIX + DIR_THREE + OM_KEY_PREFIX), - BUCKET_ONE, - VOL, - DIR_THREE, - DIR_THREE_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - getBucketLayout()); - writeDirToOm(reconOMMetadataManager, - (DIR_ONE + OM_KEY_PREFIX + DIR_FOUR + OM_KEY_PREFIX), - BUCKET_ONE, - VOL, - DIR_FOUR, - DIR_FOUR_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - getBucketLayout()); - writeDirToOm(reconOMMetadataManager, - (DIR_FIVE + OM_KEY_PREFIX), - BUCKET_THREE, - VOL_TWO, - DIR_FIVE, - DIR_FIVE_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - getBucketLayout()); - - // write all keys - writeKeyToOm(reconOMMetadataManager, - KEY_ONE, - BUCKET_ONE, - VOL, - FILE_ONE, - KEY_ONE_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - KEY_ONE_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_TWO, - BUCKET_ONE, - VOL, - FILE_TWO, - KEY_TWO_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - KEY_TWO_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_THREE, - BUCKET_ONE, - VOL, - FILE_THREE, - KEY_THREE_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - KEY_THREE_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_FOUR, - BUCKET_TWO, - VOL, - FILE_FOUR, - KEY_FOUR_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_TWO_OBJECT_ID, - VOL_OBJECT_ID, - KEY_FOUR_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_FIVE, - BUCKET_TWO, - VOL, - FILE_FIVE, - KEY_FIVE_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_TWO_OBJECT_ID, - VOL_OBJECT_ID, - KEY_FIVE_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_SIX, - BUCKET_ONE, - VOL, - FILE_SIX, - KEY_SIX_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - KEY_SIX_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_EIGHT, - BUCKET_THREE, - VOL_TWO, - FILE_EIGHT, - KEY_EIGHT_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - KEY_EIGHT_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_NINE, - BUCKET_THREE, - VOL_TWO, - FILE_NINE, - KEY_NINE_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - KEY_NINE_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_TEN, - BUCKET_THREE, - VOL_TWO, - FILE_TEN, - KEY_TEN_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - KEY_TEN_SIZE, - getBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_ELEVEN, - BUCKET_FOUR, - VOL_TWO, - FILE_ELEVEN, - KEY_ELEVEN_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_FOUR_OBJECT_ID, - VOL_TWO_OBJECT_ID, - KEY_ELEVEN_SIZE, - getBucketLayout()); - } - - private void populateVolumeThree() throws IOException { - - writeDirToOm(reconOMMetadataManager, - (DIR_SIX + OM_KEY_PREFIX), - BUCKET_FIVE, - VOL_THREE, - DIR_SIX, - DIR_SIX_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_FIVE_OBJECT_ID, - VOL_THREE_OBJECT_ID, - getBucketLayout()); - - writeDirToOm(reconOMMetadataManager, - (DIR_SIX + OM_KEY_PREFIX + DIR_SEVEN + OM_KEY_PREFIX), - BUCKET_FIVE, - VOL_THREE, - DIR_SEVEN, - DIR_SEVEN_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_FIVE_OBJECT_ID, - VOL_THREE_OBJECT_ID, - getBucketLayout()); - - writeKeyToOm(reconOMMetadataManager, - KEY_TWELVE, - BUCKET_FIVE, - VOL_THREE, - FILE_TWELVE, - FILE_TWELVE_OBJECT_ID, - BUCKET_FIVE_OBJECT_ID, - BUCKET_FIVE_OBJECT_ID, - VOL_THREE_OBJECT_ID, - KEY_TWELVE_SIZE, - getBucketLayout(), RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE), - Time.now(), true); - - writeKeyToOm(reconOMMetadataManager, - KEY_THIRTEEN, - BUCKET_FIVE, - VOL_THREE, - FILE_THIRTEEN, - FILE_THIRTEEN_OBJECT_ID, - DIR_SIX_OBJECT_ID, - BUCKET_FIVE_OBJECT_ID, - VOL_THREE_OBJECT_ID, - KEY_THIRTEEN_SIZE, - getBucketLayout(), RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE), - Time.now(), true); - - writeKeyToOm(reconOMMetadataManager, - KEY_FOURTEEN, - BUCKET_FIVE, - VOL_THREE, - FILE_FOURTEEN, - FILE_FOURTEEN_OBJECT_ID, - DIR_SEVEN_OBJECT_ID, - BUCKET_FIVE_OBJECT_ID, - VOL_THREE_OBJECT_ID, - KEY_FOURTEEN_SIZE, - getBucketLayout(), RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE), - Time.now(), true); - } - - /** - * Create a new OM Metadata manager instance with one user, one vol, and two - * buckets. - * @throws IOException ioEx - */ - private static OMMetadataManager initializeNewOmMetadataManager( - File omDbDir, OzoneConfiguration omConfiguration) - throws IOException { - omConfiguration.set(OZONE_OM_DB_DIRS, - omDbDir.getAbsolutePath()); - omConfiguration.set(OMConfigKeys - .OZONE_OM_ENABLE_FILESYSTEM_PATHS, "true"); - setConfiguration(omConfiguration); - OMMetadataManager omMetadataManager = new OmMetadataManagerImpl( - omConfiguration, null); - - String volumeKey = omMetadataManager.getVolumeKey(VOL); - OmVolumeArgs args = - OmVolumeArgs.newBuilder() - .setObjectID(VOL_OBJECT_ID) - .setVolume(VOL) - .setAdminName(TEST_USER) - .setOwnerName(TEST_USER) - .setQuotaInBytes(VOL_QUOTA) - .build(); - - String volume2Key = omMetadataManager.getVolumeKey(VOL_TWO); - OmVolumeArgs args2 = - OmVolumeArgs.newBuilder() - .setObjectID(VOL_TWO_OBJECT_ID) - .setVolume(VOL_TWO) - .setAdminName(TEST_USER) - .setOwnerName(TEST_USER) - .setQuotaInBytes(VOL_TWO_QUOTA) - .build(); - - String volume3Key = omMetadataManager.getVolumeKey(VOL_THREE); - OmVolumeArgs args3 = - OmVolumeArgs.newBuilder() - .setObjectID(VOL_THREE_OBJECT_ID) - .setVolume(VOL_THREE) - .setAdminName(TEST_USER) - .setOwnerName(TEST_USER) - .setQuotaInBytes(VOL_THREE_QUOTA) - .build(); - - omMetadataManager.getVolumeTable().put(volumeKey, args); - omMetadataManager.getVolumeTable().put(volume2Key, args2); - omMetadataManager.getVolumeTable().put(volume3Key, args3); - - OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() - .setVolumeName(VOL) - .setBucketName(BUCKET_ONE) - .setObjectID(BUCKET_ONE_OBJECT_ID) - .setQuotaInBytes(BUCKET_ONE_QUOTA) - .setBucketLayout(getBucketLayout()) - .build(); - - OmBucketInfo bucketInfo2 = OmBucketInfo.newBuilder() - .setVolumeName(VOL) - .setBucketName(BUCKET_TWO) - .setObjectID(BUCKET_TWO_OBJECT_ID) - .setQuotaInBytes(BUCKET_TWO_QUOTA) - .setBucketLayout(getBucketLayout()) - .build(); - - OmBucketInfo bucketInfo3 = OmBucketInfo.newBuilder() - .setVolumeName(VOL_TWO) - .setBucketName(BUCKET_THREE) - .setObjectID(BUCKET_THREE_OBJECT_ID) - .setQuotaInBytes(BUCKET_THREE_QUOTA) - .setBucketLayout(getBucketLayout()) - .build(); - - OmBucketInfo bucketInfo4 = OmBucketInfo.newBuilder() - .setVolumeName(VOL_TWO) - .setBucketName(BUCKET_FOUR) - .setObjectID(BUCKET_FOUR_OBJECT_ID) - .setQuotaInBytes(BUCKET_FOUR_QUOTA) - .setBucketLayout(getBucketLayout()) - .build(); - - OmBucketInfo bucketInfo5 = OmBucketInfo.newBuilder() - .setVolumeName(VOL_THREE) - .setBucketName(BUCKET_FIVE) - .setObjectID(BUCKET_FIVE_OBJECT_ID) - .setQuotaInBytes(BUCKET_FIVE_QUOTA) - .setBucketLayout(getBucketLayout()) - .build(); - - String bucketKey = omMetadataManager.getBucketKey( - bucketInfo.getVolumeName(), bucketInfo.getBucketName()); - String bucketKey2 = omMetadataManager.getBucketKey( - bucketInfo2.getVolumeName(), bucketInfo2.getBucketName()); - String bucketKey3 = omMetadataManager.getBucketKey( - bucketInfo3.getVolumeName(), bucketInfo3.getBucketName()); - String bucketKey4 = omMetadataManager.getBucketKey( - bucketInfo4.getVolumeName(), bucketInfo4.getBucketName()); - String bucketKey5 = omMetadataManager.getBucketKey( - bucketInfo5.getVolumeName(), bucketInfo5.getBucketName()); - - omMetadataManager.getBucketTable().put(bucketKey, bucketInfo); - omMetadataManager.getBucketTable().put(bucketKey2, bucketInfo2); - omMetadataManager.getBucketTable().put(bucketKey3, bucketInfo3); - omMetadataManager.getBucketTable().put(bucketKey4, bucketInfo4); - omMetadataManager.getBucketTable().put(bucketKey5, bucketInfo5); - - return omMetadataManager; - } - - private void setUpMultiBlockKey() throws IOException { - OmKeyLocationInfoGroup locationInfoGroup = - getLocationInfoGroup1(); - - // add the multi-block key to Recon's OM - writeKeyToOm(reconOMMetadataManager, - MULTI_BLOCK_KEY, - BUCKET_ONE, - VOL, - MULTI_BLOCK_FILE, - MULTI_BLOCK_KEY_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup), - getBucketLayout(), - KEY_SEVEN_SIZE); - } - - private OmKeyLocationInfoGroup getLocationInfoGroup1() { - List locationInfoList = new ArrayList<>(); - BlockID block1 = new BlockID(CONTAINER_ONE_ID, 0L); - BlockID block2 = new BlockID(CONTAINER_TWO_ID, 0L); - BlockID block3 = new BlockID(CONTAINER_THREE_ID, 0L); - - OmKeyLocationInfo location1 = new OmKeyLocationInfo.Builder() - .setBlockID(block1) - .setLength(BLOCK_ONE_LENGTH) - .build(); - OmKeyLocationInfo location2 = new OmKeyLocationInfo.Builder() - .setBlockID(block2) - .setLength(BLOCK_TWO_LENGTH) - .build(); - OmKeyLocationInfo location3 = new OmKeyLocationInfo.Builder() - .setBlockID(block3) - .setLength(BLOCK_THREE_LENGTH) - .build(); - locationInfoList.add(location1); - locationInfoList.add(location2); - locationInfoList.add(location3); - - return new OmKeyLocationInfoGroup(0L, locationInfoList); - } - - /** - * Testing the following case. - * vol - * / \ - * bucket1 bucket2 - * / \ / \ - * file1 dir1 file4 file5 - * / \ \ \ - * dir2 dir3 dir4 file7 - * / \ \ - * file2 file3 file6 - * ---------------------------------------- - * vol2 - * / \ - * bucket3 bucket4 - * / \ / - * file8 dir5 file11 - * / \ - * file9 file10 - * Write these keys to OM and - * replicate them. - */ - private OmKeyLocationInfoGroup getLocationInfoGroup2() { - List locationInfoList = new ArrayList<>(); - BlockID block4 = new BlockID(CONTAINER_FOUR_ID, 0L); - BlockID block5 = new BlockID(CONTAINER_FIVE_ID, 0L); - BlockID block6 = new BlockID(CONTAINER_SIX_ID, 0L); - - OmKeyLocationInfo location4 = new OmKeyLocationInfo.Builder() - .setBlockID(block4) - .setLength(BLOCK_FOUR_LENGTH) - .build(); - OmKeyLocationInfo location5 = new OmKeyLocationInfo.Builder() - .setBlockID(block5) - .setLength(BLOCK_FIVE_LENGTH) - .build(); - OmKeyLocationInfo location6 = new OmKeyLocationInfo.Builder() - .setBlockID(block6) - .setLength(BLOCK_SIX_LENGTH) - .build(); - locationInfoList.add(location4); - locationInfoList.add(location5); - locationInfoList.add(location6); - return new OmKeyLocationInfoGroup(0L, locationInfoList); - - } - - @SuppressWarnings("checkstyle:MethodLength") - private void setUpMultiBlockReplicatedKeys() throws IOException { - OmKeyLocationInfoGroup locationInfoGroup1 = - getLocationInfoGroup1(); - OmKeyLocationInfoGroup locationInfoGroup2 = - getLocationInfoGroup2(); - - //vol/bucket1/file1 - writeKeyToOm(reconOMMetadataManager, - KEY_ONE, - BUCKET_ONE, - VOL, - FILE_ONE, - KEY_ONE_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getBucketLayout(), - KEY_ONE_SIZE); - - //vol/bucket1/dir1/dir2/file2 - writeKeyToOm(reconOMMetadataManager, - KEY_TWO, - BUCKET_ONE, - VOL, - FILE_TWO, - KEY_TWO_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup2), - getBucketLayout(), - KEY_TWO_SIZE); - - //vol/bucket1/dir1/dir3/file3 - writeKeyToOm(reconOMMetadataManager, - KEY_THREE, - BUCKET_ONE, - VOL, - FILE_THREE, - KEY_THREE_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getBucketLayout(), - KEY_THREE_SIZE); - - //vol/bucket2/file4 - writeKeyToOm(reconOMMetadataManager, - KEY_FOUR, - BUCKET_TWO, - VOL, - FILE_FOUR, - KEY_FOUR_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_TWO_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup2), - getBucketLayout(), - KEY_FOUR_SIZE); - - //vol/bucket2/file5 - writeKeyToOm(reconOMMetadataManager, - KEY_FIVE, - BUCKET_TWO, - VOL, - FILE_FIVE, - KEY_FIVE_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_TWO_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getBucketLayout(), - KEY_FIVE_SIZE); - - //vol/bucket1/dir1/dir4/file6 - writeKeyToOm(reconOMMetadataManager, - KEY_SIX, - BUCKET_ONE, - VOL, - FILE_SIX, - KEY_SIX_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup2), - getBucketLayout(), - KEY_SIX_SIZE); - - //vol/bucket1/dir1/file7 - writeKeyToOm(reconOMMetadataManager, - KEY_SEVEN, - BUCKET_ONE, - VOL, - FILE_SEVEN, - KEY_SEVEN_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getBucketLayout(), - KEY_SEVEN_SIZE); - - //vol2/bucket3/file8 - writeKeyToOm(reconOMMetadataManager, - KEY_EIGHT, - BUCKET_THREE, - VOL_TWO, - FILE_EIGHT, - KEY_EIGHT_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - Collections.singletonList(locationInfoGroup2), - getBucketLayout(), - KEY_EIGHT_SIZE); - - //vol2/bucket3/dir5/file9 - writeKeyToOm(reconOMMetadataManager, - KEY_NINE, - BUCKET_THREE, - VOL_TWO, - FILE_NINE, - KEY_NINE_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getBucketLayout(), - KEY_NINE_SIZE); - - //vol2/bucket3/dir5/file10 - writeKeyToOm(reconOMMetadataManager, - KEY_TEN, - BUCKET_THREE, - VOL_TWO, - FILE_TEN, - KEY_TEN_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - Collections.singletonList(locationInfoGroup2), - getBucketLayout(), - KEY_TEN_SIZE); - - //vol2/bucket4/file11 - writeKeyToOm(reconOMMetadataManager, - KEY_ELEVEN, - BUCKET_FOUR, - VOL_TWO, - FILE_ELEVEN, - KEY_ELEVEN_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_FOUR_OBJECT_ID, - VOL_TWO_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getBucketLayout(), - KEY_ELEVEN_SIZE); - } - - /** - * Generate a set of mock container replica with a size of - * replication factor for container. - * @param replicationFactor number of replica - * @param containerID the container replicated based upon - * @return a set of container replica for testing - */ - private static Set generateMockContainerReplicas( - int replicationFactor, ContainerID containerID) { - Set result = new HashSet<>(); - for (int i = 0; i < replicationFactor; ++i) { - DatanodeDetails randomDatanode = randomDatanodeDetails(); - ContainerReplica replica = new ContainerReplica.ContainerReplicaBuilder() - .setContainerID(containerID) - .setContainerState(State.OPEN) - .setDatanodeDetails(randomDatanode) - .build(); - result.add(replica); - } - return result; - } - - private static ReconStorageContainerManagerFacade getMockReconSCM() - throws ContainerNotFoundException { - ReconStorageContainerManagerFacade reconSCM = - mock(ReconStorageContainerManagerFacade.class); - ContainerManager containerManager = mock(ContainerManager.class); - - // Container 1 is 3-way replicated - ContainerID containerID1 = ContainerID.valueOf(CONTAINER_ONE_ID); - Set containerReplicas1 = generateMockContainerReplicas( - CONTAINER_ONE_REPLICA_COUNT, containerID1); - when(containerManager.getContainerReplicas(containerID1)) - .thenReturn(containerReplicas1); - - // Container 2 is under replicated with 2 replica - ContainerID containerID2 = ContainerID.valueOf(CONTAINER_TWO_ID); - Set containerReplicas2 = generateMockContainerReplicas( - CONTAINER_TWO_REPLICA_COUNT, containerID2); - when(containerManager.getContainerReplicas(containerID2)) - .thenReturn(containerReplicas2); - - // Container 3 is over replicated with 4 replica - ContainerID containerID3 = ContainerID.valueOf(CONTAINER_THREE_ID); - Set containerReplicas3 = generateMockContainerReplicas( - CONTAINER_THREE_REPLICA_COUNT, containerID3); - when(containerManager.getContainerReplicas(containerID3)) - .thenReturn(containerReplicas3); - - // Container 4 is replicated with 5 replica - ContainerID containerID4 = ContainerID.valueOf(CONTAINER_FOUR_ID); - Set containerReplicas4 = generateMockContainerReplicas( - CONTAINER_FOUR_REPLICA_COUNT, containerID4); - when(containerManager.getContainerReplicas(containerID4)) - .thenReturn(containerReplicas4); - - // Container 5 is replicated with 2 replica - ContainerID containerID5 = ContainerID.valueOf(CONTAINER_FIVE_ID); - Set containerReplicas5 = generateMockContainerReplicas( - CONTAINER_FIVE_REPLICA_COUNT, containerID5); - when(containerManager.getContainerReplicas(containerID5)) - .thenReturn(containerReplicas5); - - // Container 6 is replicated with 3 replica - ContainerID containerID6 = ContainerID.valueOf(CONTAINER_SIX_ID); - Set containerReplicas6 = generateMockContainerReplicas( - CONTAINER_SIX_REPLICA_COUNT, containerID6); - when(containerManager.getContainerReplicas(containerID6)) - .thenReturn(containerReplicas6); - - when(reconSCM.getContainerManager()).thenReturn(containerManager); - ReconNodeManager mockReconNodeManager = mock(ReconNodeManager.class); - when(mockReconNodeManager.getStats()).thenReturn(getMockSCMRootStat()); - when(reconSCM.getScmNodeManager()).thenReturn(mockReconNodeManager); - return reconSCM; - } - - private static BucketLayout getBucketLayout() { - return BucketLayout.LEGACY; - } - - private static SCMNodeStat getMockSCMRootStat() { - return new SCMNodeStat(ROOT_QUOTA, ROOT_DATA_SIZE, - ROOT_QUOTA - ROOT_DATA_SIZE, 0, ROOT_QUOTA - ROOT_DATA_SIZE - 1, 0); - } -} diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithOBSAndLegacy.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithOBSAndLegacy.java deleted file mode 100644 index ecb08c132340..000000000000 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithOBSAndLegacy.java +++ /dev/null @@ -1,1452 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.recon.api; - -import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS; -import static org.apache.hadoop.ozone.om.helpers.QuotaUtil.getReplicatedSize; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getMockOzoneManagerServiceProvider; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import javax.ws.rs.core.Response; -import org.apache.hadoop.hdds.client.BlockID; -import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.StorageType; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos; -import org.apache.hadoop.hdds.scm.container.ContainerID; -import org.apache.hadoop.hdds.scm.container.ContainerManager; -import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; -import org.apache.hadoop.hdds.scm.container.ContainerReplica; -import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat; -import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; -import org.apache.hadoop.ozone.OmUtils; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.om.OMConfigKeys; -import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; -import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; -import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; -import org.apache.hadoop.ozone.recon.ReconConstants; -import org.apache.hadoop.ozone.recon.ReconTestInjector; -import org.apache.hadoop.ozone.recon.ReconUtils; -import org.apache.hadoop.ozone.recon.api.handlers.BucketHandler; -import org.apache.hadoop.ozone.recon.api.handlers.EntityHandler; -import org.apache.hadoop.ozone.recon.api.types.BucketObjectDBInfo; -import org.apache.hadoop.ozone.recon.api.types.DUResponse; -import org.apache.hadoop.ozone.recon.api.types.EntityType; -import org.apache.hadoop.ozone.recon.api.types.FileSizeDistributionResponse; -import org.apache.hadoop.ozone.recon.api.types.NamespaceSummaryResponse; -import org.apache.hadoop.ozone.recon.api.types.QuotaUsageResponse; -import org.apache.hadoop.ozone.recon.api.types.ResponseStatus; -import org.apache.hadoop.ozone.recon.api.types.VolumeObjectDBInfo; -import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; -import org.apache.hadoop.ozone.recon.scm.ReconNodeManager; -import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade; -import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; -import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; -import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; -import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; -import org.apache.hadoop.ozone.recon.tasks.NSSummaryTaskWithLegacy; -import org.apache.hadoop.ozone.recon.tasks.NSSummaryTaskWithOBS; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -/** - * Tests the NSSummary REST APIs within the context of an Object Store (OBS) layout, - * as well as Legacy layout buckets with FileSystemPaths disabled. The tests aim to - * validate API responses for buckets that follow the flat hierarchy model typical - * of OBS layouts. - *

- * The test environment simulates a simple object storage structure with volumes - * containing buckets, which in turn contain files. Specifically, it includes: - * - Two OBS layout buckets (bucket1 and bucket2) under 'vol', each containing - * multiple files. - * - Two Legacy layout buckets (bucket3 and bucket4) under 'vol2', with 'bucket4' - * the fileSystemEnabled flag set to false for these legacy buckets. - *

- * The directory structure for testing is as follows: - * . - * └── vol - * ├── bucket1 (OBS) - * │ ├── KEY_ONE - * │ ├── KEY_TWO - * │ └── KEY_THREE - * └── bucket2 (OBS) - * ├── KEY_FOUR - * └── KEY_FIVE - * └── vol2 - * ├── bucket3 (Legacy) - * │ ├── KEY_EIGHT - * │ ├── KEY_NINE - * │ └── KEY_TEN - * └── bucket4 (Legacy) - * └── KEY_ELEVEN - */ -public class TestNSSummaryEndpointWithOBSAndLegacy extends NSSummaryTests { - @TempDir - private Path temporaryFolder; - - private ReconOMMetadataManager reconOMMetadataManager; - private ReconNamespaceSummaryManager reconNamespaceSummaryManager; - private NSSummaryEndpoint nsSummaryEndpoint; - - private static final String TEST_PATH_UTILITY = - "/vol1/buck1/a/b/c/d/e/file1.txt"; - private static final String PARENT_DIR = "vol1/buck1/a/b/c/d/e"; - private static final String[] TEST_NAMES = - new String[]{"vol1", "buck1", "a", "b", "c", "d", "e", "file1.txt"}; - private static final String TEST_KEY_NAMES = "a/b/c/d/e/file1.txt"; - - // Object names - private static final String VOL = "vol"; - private static final String VOL_TWO = "vol2"; - private static final String BUCKET_ONE = "bucket1"; - private static final String BUCKET_TWO = "bucket2"; - private static final String BUCKET_THREE = "bucket3"; - private static final String BUCKET_FOUR = "bucket4"; - private static final String KEY_ONE = "file1"; - private static final String KEY_TWO = "////file2"; - private static final String KEY_THREE = "file3///"; - private static final String KEY_FOUR = "file4"; - private static final String KEY_FIVE = "_//////"; - private static final String KEY_EIGHT = "file8"; - private static final String KEY_NINE = "//////"; - private static final String KEY_TEN = "///__file10"; - private static final String KEY_ELEVEN = "////file11"; - private static final String MULTI_BLOCK_FILE = KEY_THREE; - - private static final long PARENT_OBJECT_ID_ZERO = 0L; - private static final long VOL_OBJECT_ID = 0L; - private static final long VOL_TWO_OBJECT_ID = 14L; - private static final long BUCKET_ONE_OBJECT_ID = 1L; - private static final long BUCKET_TWO_OBJECT_ID = 2L; - private static final long BUCKET_THREE_OBJECT_ID = 15L; - private static final long BUCKET_FOUR_OBJECT_ID = 16L; - private static final long KEY_ONE_OBJECT_ID = 3L; - private static final long KEY_TWO_OBJECT_ID = 5L; - private static final long KEY_THREE_OBJECT_ID = 8L; - private static final long KEY_FOUR_OBJECT_ID = 6L; - private static final long KEY_FIVE_OBJECT_ID = 9L; - private static final long KEY_EIGHT_OBJECT_ID = 17L; - private static final long KEY_NINE_OBJECT_ID = 19L; - private static final long KEY_TEN_OBJECT_ID = 20L; - private static final long KEY_ELEVEN_OBJECT_ID = 21L; - private static final long MULTI_BLOCK_KEY_OBJECT_ID = 13L; - - // container IDs - private static final long CONTAINER_ONE_ID = 1L; - private static final long CONTAINER_TWO_ID = 2L; - private static final long CONTAINER_THREE_ID = 3L; - private static final long CONTAINER_FOUR_ID = 4L; - private static final long CONTAINER_FIVE_ID = 5L; - private static final long CONTAINER_SIX_ID = 6L; - - // replication factors - private static final int CONTAINER_ONE_REPLICA_COUNT = 3; - private static final int CONTAINER_TWO_REPLICA_COUNT = 2; - private static final int CONTAINER_THREE_REPLICA_COUNT = 4; - private static final int CONTAINER_FOUR_REPLICA_COUNT = 5; - private static final int CONTAINER_FIVE_REPLICA_COUNT = 2; - private static final int CONTAINER_SIX_REPLICA_COUNT = 3; - - // block lengths - private static final long BLOCK_ONE_LENGTH = 1000L; - private static final long BLOCK_TWO_LENGTH = 2000L; - private static final long BLOCK_THREE_LENGTH = 3000L; - private static final long BLOCK_FOUR_LENGTH = 4000L; - private static final long BLOCK_FIVE_LENGTH = 5000L; - private static final long BLOCK_SIX_LENGTH = 6000L; - - // data size in bytes - private static final long FILE_ONE_SIZE = 500L; // bin 0 - private static final long FILE_TWO_SIZE = OzoneConsts.KB + 1; // bin 1 - private static final long FILE_THREE_SIZE = 4 * OzoneConsts.KB + 1; // bin 3 - private static final long FILE_FOUR_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 - private static final long FILE_FIVE_SIZE = 100L; // bin 0 - private static final long FILE_EIGHT_SIZE = OzoneConsts.KB + 1; // bin 1 - private static final long FILE_NINE_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 - private static final long FILE_TEN_SIZE = 2 * OzoneConsts.KB + 1; // bin 2 - private static final long FILE_ELEVEN_SIZE = OzoneConsts.KB + 1; // bin 1 - - private static final long FILE1_SIZE_WITH_REPLICA = - getReplicatedSize(FILE_ONE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE2_SIZE_WITH_REPLICA = - getReplicatedSize(FILE_TWO_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE3_SIZE_WITH_REPLICA = - getReplicatedSize(FILE_THREE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE4_SIZE_WITH_REPLICA = - getReplicatedSize(FILE_FOUR_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE5_SIZE_WITH_REPLICA = - getReplicatedSize(FILE_FIVE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - - private static final long FILE8_SIZE_WITH_REPLICA = - getReplicatedSize(FILE_EIGHT_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE9_SIZE_WITH_REPLICA = - getReplicatedSize(FILE_NINE_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE10_SIZE_WITH_REPLICA = - getReplicatedSize(FILE_TEN_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - private static final long FILE11_SIZE_WITH_REPLICA = - getReplicatedSize(FILE_ELEVEN_SIZE, - StandaloneReplicationConfig.getInstance(ONE)); - - private static final long MULTI_BLOCK_KEY_SIZE_WITH_REPLICA - = FILE3_SIZE_WITH_REPLICA; - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_ROOT - = FILE1_SIZE_WITH_REPLICA - + FILE2_SIZE_WITH_REPLICA - + FILE3_SIZE_WITH_REPLICA - + FILE4_SIZE_WITH_REPLICA - + FILE5_SIZE_WITH_REPLICA - + FILE8_SIZE_WITH_REPLICA - + FILE9_SIZE_WITH_REPLICA - + FILE10_SIZE_WITH_REPLICA - + FILE11_SIZE_WITH_REPLICA; - - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_VOL - = FILE1_SIZE_WITH_REPLICA - + FILE2_SIZE_WITH_REPLICA - + FILE3_SIZE_WITH_REPLICA - + FILE4_SIZE_WITH_REPLICA - + FILE5_SIZE_WITH_REPLICA; - - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET1 - = FILE1_SIZE_WITH_REPLICA - + FILE2_SIZE_WITH_REPLICA - + FILE3_SIZE_WITH_REPLICA; - - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET3 - = FILE8_SIZE_WITH_REPLICA + - FILE9_SIZE_WITH_REPLICA + - FILE10_SIZE_WITH_REPLICA; - - private static final long - MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_KEY - = FILE4_SIZE_WITH_REPLICA; - - // quota in bytes - private static final long ROOT_QUOTA = 2 * (2 * OzoneConsts.MB); - private static final long VOL_QUOTA = 2 * OzoneConsts.MB; - private static final long VOL_TWO_QUOTA = 2 * OzoneConsts.MB; - private static final long BUCKET_ONE_QUOTA = OzoneConsts.MB; - private static final long BUCKET_TWO_QUOTA = OzoneConsts.MB; - private static final long BUCKET_THREE_QUOTA = OzoneConsts.MB; - private static final long BUCKET_FOUR_QUOTA = OzoneConsts.MB; - - // mock client's path requests - private static final String TEST_USER = "TestUser"; - private static final String ROOT_PATH = "/"; - private static final String VOL_PATH = ROOT_PATH + VOL; - private static final String VOL_TWO_PATH = ROOT_PATH + VOL_TWO; - private static final String BUCKET_ONE_PATH = - ROOT_PATH + VOL + ROOT_PATH + BUCKET_ONE; - private static final String BUCKET_TWO_PATH = - ROOT_PATH + VOL + ROOT_PATH + BUCKET_TWO; - private static final String BUCKET_THREE_PATH = - ROOT_PATH + VOL_TWO + ROOT_PATH + BUCKET_THREE; - private static final String BUCKET_FOUR_PATH = - ROOT_PATH + VOL_TWO + ROOT_PATH + BUCKET_FOUR; - private static final String KEY_ONE_PATH = - ROOT_PATH + VOL + ROOT_PATH + BUCKET_ONE + ROOT_PATH + KEY_ONE; - private static final String KEY_TWO_PATH = - ROOT_PATH + VOL + ROOT_PATH + BUCKET_ONE + ROOT_PATH + KEY_TWO; - private static final String KEY_FIVE_PATH = - ROOT_PATH + VOL + ROOT_PATH + BUCKET_TWO + ROOT_PATH + KEY_FIVE; - private static final String KEY_EIGHT_PATH = - ROOT_PATH + VOL_TWO + ROOT_PATH + BUCKET_THREE + ROOT_PATH + KEY_EIGHT; - private static final String KEY_ELEVEN_PATH = - ROOT_PATH + VOL_TWO + ROOT_PATH + BUCKET_FOUR + ROOT_PATH + KEY_ELEVEN; - private static final String KEY4_PATH = - ROOT_PATH + VOL + ROOT_PATH + BUCKET_TWO + ROOT_PATH + KEY_FOUR; - private static final String MULTI_BLOCK_KEY_PATH = - ROOT_PATH + VOL + ROOT_PATH + BUCKET_ONE + ROOT_PATH + KEY_THREE; - private static final String INVALID_PATH = "/vol/path/not/found"; - - // some expected answers - private static final long ROOT_DATA_SIZE = - FILE_ONE_SIZE + FILE_TWO_SIZE + FILE_THREE_SIZE + FILE_FOUR_SIZE + - FILE_FIVE_SIZE + FILE_EIGHT_SIZE + FILE_NINE_SIZE + FILE_TEN_SIZE + - FILE_ELEVEN_SIZE; - private static final long VOL_DATA_SIZE = FILE_ONE_SIZE + FILE_TWO_SIZE + - FILE_THREE_SIZE + FILE_FOUR_SIZE + FILE_FIVE_SIZE; - - private static final long VOL_TWO_DATA_SIZE = - FILE_EIGHT_SIZE + FILE_NINE_SIZE + FILE_TEN_SIZE + FILE_ELEVEN_SIZE; - - private static final long BUCKET_ONE_DATA_SIZE = FILE_ONE_SIZE + - FILE_TWO_SIZE + - FILE_THREE_SIZE; - - private static final long BUCKET_TWO_DATA_SIZE = - FILE_FOUR_SIZE + FILE_FIVE_SIZE; - - private static final long BUCKET_THREE_DATA_SIZE = - FILE_EIGHT_SIZE + FILE_NINE_SIZE + FILE_TEN_SIZE; - - private static final long BUCKET_FOUR_DATA_SIZE = FILE_ELEVEN_SIZE; - - @BeforeEach - public void setUp() throws Exception { - OzoneConfiguration conf = new OzoneConfiguration(); - // By setting this config our Legacy buckets will behave like OBS buckets. - conf.set(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, "false"); - OMMetadataManager omMetadataManager = initializeNewOmMetadataManager( - Files.createDirectory(temporaryFolder.resolve( - "JunitOmDBDir")).toFile(), conf); - OzoneManagerServiceProviderImpl ozoneManagerServiceProvider = - getMockOzoneManagerServiceProvider(); - reconOMMetadataManager = getTestReconOmMetadataManager(omMetadataManager, - Files.createDirectory(temporaryFolder.resolve( - "omMetadatDir")).toFile()); - - ReconTestInjector reconTestInjector = - new ReconTestInjector.Builder(temporaryFolder.toFile()) - .withReconOm(reconOMMetadataManager) - .withOmServiceProvider(ozoneManagerServiceProvider) - .withReconSqlDb() - .withContainerDB() - .addBinding(OzoneStorageContainerManager.class, - getMockReconSCM()) - .addBinding(StorageContainerServiceProvider.class, - mock(StorageContainerServiceProviderImpl.class)) - .addBinding(NSSummaryEndpoint.class) - .build(); - reconNamespaceSummaryManager = - reconTestInjector.getInstance(ReconNamespaceSummaryManager.class); - nsSummaryEndpoint = reconTestInjector.getInstance(NSSummaryEndpoint.class); - - // populate OM DB and reprocess into Recon RocksDB - populateOMDB(); - NSSummaryTaskWithOBS nsSummaryTaskWithOBS = - new NSSummaryTaskWithOBS(reconNamespaceSummaryManager, - reconOMMetadataManager, 10, 5, 20, 2000); - nsSummaryTaskWithOBS.reprocessWithOBS(reconOMMetadataManager); - NSSummaryTaskWithLegacy nsSummaryTaskWithLegacy = - new NSSummaryTaskWithLegacy(reconNamespaceSummaryManager, - reconOMMetadataManager, conf, 10); - nsSummaryTaskWithLegacy.reprocessWithLegacy(reconOMMetadataManager); - } - - @Test - public void testUtility() { - String[] names = EntityHandler.parseRequestPath(TEST_PATH_UTILITY); - assertArrayEquals(TEST_NAMES, names); - String keyName = BucketHandler.getKeyName(names); - assertEquals(TEST_KEY_NAMES, keyName); - String subpath = BucketHandler.buildSubpath(PARENT_DIR, "file1.txt"); - assertEquals(TEST_PATH_UTILITY, subpath); - } - - @Test - public void testGetBasicInfoRoot() throws Exception { - // Test root basics - Response rootResponse = nsSummaryEndpoint.getBasicInfo(ROOT_PATH); - NamespaceSummaryResponse rootResponseObj = - (NamespaceSummaryResponse) rootResponse.getEntity(); - assertEquals(EntityType.ROOT, rootResponseObj.getEntityType()); - assertEquals(2, rootResponseObj.getCountStats().getNumVolume()); - assertEquals(4, rootResponseObj.getCountStats().getNumBucket()); - assertEquals(9, rootResponseObj.getCountStats().getNumTotalKey()); - } - - @Test - public void testGetBasicInfoVol() throws Exception { - // Test volume basics - Response volResponse = nsSummaryEndpoint.getBasicInfo(VOL_PATH); - NamespaceSummaryResponse volResponseObj = - (NamespaceSummaryResponse) volResponse.getEntity(); - assertEquals(EntityType.VOLUME, - volResponseObj.getEntityType()); - assertEquals(2, volResponseObj.getCountStats().getNumBucket()); - assertEquals(5, volResponseObj.getCountStats().getNumTotalKey()); - assertEquals(TEST_USER, ((VolumeObjectDBInfo) volResponseObj. - getObjectDBInfo()).getAdmin()); - assertEquals(TEST_USER, ((VolumeObjectDBInfo) volResponseObj. - getObjectDBInfo()).getOwner()); - assertEquals(VOL, volResponseObj.getObjectDBInfo().getName()); - assertEquals(2097152, volResponseObj.getObjectDBInfo().getQuotaInBytes()); - assertEquals(-1, volResponseObj.getObjectDBInfo().getQuotaInNamespace()); - } - - @Test - public void testGetBasicInfoVolTwo() throws Exception { - // Test volume 2's basics - Response volTwoResponse = nsSummaryEndpoint.getBasicInfo(VOL_TWO_PATH); - NamespaceSummaryResponse volTwoResponseObj = - (NamespaceSummaryResponse) volTwoResponse.getEntity(); - assertEquals(EntityType.VOLUME, - volTwoResponseObj.getEntityType()); - assertEquals(2, volTwoResponseObj.getCountStats().getNumBucket()); - assertEquals(4, volTwoResponseObj.getCountStats().getNumTotalKey()); - assertEquals(TEST_USER, ((VolumeObjectDBInfo) volTwoResponseObj. - getObjectDBInfo()).getAdmin()); - assertEquals(TEST_USER, ((VolumeObjectDBInfo) volTwoResponseObj. - getObjectDBInfo()).getOwner()); - assertEquals(VOL_TWO, volTwoResponseObj.getObjectDBInfo().getName()); - assertEquals(2097152, - volTwoResponseObj.getObjectDBInfo().getQuotaInBytes()); - assertEquals(-1, volTwoResponseObj.getObjectDBInfo().getQuotaInNamespace()); - } - - @Test - public void testGetBasicInfoBucketOne() throws Exception { - // Test bucket 1's basics - Response bucketOneResponse = - nsSummaryEndpoint.getBasicInfo(BUCKET_ONE_PATH); - NamespaceSummaryResponse bucketOneObj = - (NamespaceSummaryResponse) bucketOneResponse.getEntity(); - assertEquals(EntityType.BUCKET, bucketOneObj.getEntityType()); - assertEquals(3, bucketOneObj.getCountStats().getNumTotalKey()); - assertEquals(VOL, - ((BucketObjectDBInfo) bucketOneObj.getObjectDBInfo()).getVolumeName()); - assertEquals(StorageType.DISK, - ((BucketObjectDBInfo) - bucketOneObj.getObjectDBInfo()).getStorageType()); - assertEquals(getOBSBucketLayout(), - ((BucketObjectDBInfo) - bucketOneObj.getObjectDBInfo()).getBucketLayout()); - assertEquals(BUCKET_ONE, - ((BucketObjectDBInfo) bucketOneObj.getObjectDBInfo()).getName()); - } - - @Test - public void testGetBasicInfoBucketTwo() throws Exception { - // Test bucket 2's basics - Response bucketTwoResponse = - nsSummaryEndpoint.getBasicInfo(BUCKET_TWO_PATH); - NamespaceSummaryResponse bucketTwoObj = - (NamespaceSummaryResponse) bucketTwoResponse.getEntity(); - assertEquals(EntityType.BUCKET, bucketTwoObj.getEntityType()); - assertEquals(2, bucketTwoObj.getCountStats().getNumTotalKey()); - assertEquals(VOL, - ((BucketObjectDBInfo) bucketTwoObj.getObjectDBInfo()).getVolumeName()); - assertEquals(StorageType.DISK, - ((BucketObjectDBInfo) - bucketTwoObj.getObjectDBInfo()).getStorageType()); - assertEquals(getOBSBucketLayout(), - ((BucketObjectDBInfo) - bucketTwoObj.getObjectDBInfo()).getBucketLayout()); - assertEquals(BUCKET_TWO, - ((BucketObjectDBInfo) bucketTwoObj.getObjectDBInfo()).getName()); - } - - @Test - public void testGetBasicInfoBucketThree() throws Exception { - // Test bucket 3's basics - Response bucketThreeResponse = - nsSummaryEndpoint.getBasicInfo(BUCKET_THREE_PATH); - NamespaceSummaryResponse bucketThreeObj = (NamespaceSummaryResponse) - bucketThreeResponse.getEntity(); - assertEquals(EntityType.BUCKET, bucketThreeObj.getEntityType()); - assertEquals(3, bucketThreeObj.getCountStats().getNumTotalKey()); - assertEquals(VOL_TWO, - ((BucketObjectDBInfo) bucketThreeObj.getObjectDBInfo()).getVolumeName()); - assertEquals(StorageType.DISK, - ((BucketObjectDBInfo) - bucketThreeObj.getObjectDBInfo()).getStorageType()); - assertEquals(getLegacyBucketLayout(), - ((BucketObjectDBInfo) - bucketThreeObj.getObjectDBInfo()).getBucketLayout()); - assertEquals(BUCKET_THREE, - ((BucketObjectDBInfo) bucketThreeObj.getObjectDBInfo()).getName()); - } - - @Test - public void testGetBasicInfoBucketFour() throws Exception { - // Test bucket 4's basics - Response bucketFourResponse = - nsSummaryEndpoint.getBasicInfo(BUCKET_FOUR_PATH); - NamespaceSummaryResponse bucketFourObj = - (NamespaceSummaryResponse) bucketFourResponse.getEntity(); - assertEquals(EntityType.BUCKET, bucketFourObj.getEntityType()); - assertEquals(1, bucketFourObj.getCountStats().getNumTotalKey()); - assertEquals(VOL_TWO, - ((BucketObjectDBInfo) bucketFourObj.getObjectDBInfo()).getVolumeName()); - assertEquals(StorageType.DISK, - ((BucketObjectDBInfo) - bucketFourObj.getObjectDBInfo()).getStorageType()); - assertEquals(getLegacyBucketLayout(), - ((BucketObjectDBInfo) - bucketFourObj.getObjectDBInfo()).getBucketLayout()); - assertEquals(BUCKET_FOUR, - ((BucketObjectDBInfo) bucketFourObj.getObjectDBInfo()).getName()); - } - - @Test - public void testGetBasicInfoNoPath() throws Exception { - // Test invalid path - testNSSummaryBasicInfoNoPath(nsSummaryEndpoint); - } - - @Test - public void testGetBasicInfoKey() throws Exception { - // Test key - testNSSummaryBasicInfoKey(nsSummaryEndpoint); - } - - @Test - public void testDiskUsageRoot() throws Exception { - // root level DU - Response rootResponse = nsSummaryEndpoint.getDiskUsage(ROOT_PATH, - false, false, false); - DUResponse duRootRes = (DUResponse) rootResponse.getEntity(); - assertEquals(2, duRootRes.getCount()); - List duRootData = duRootRes.getDuData(); - // sort based on subpath - Collections.sort(duRootData, - Comparator.comparing(DUResponse.DiskUsage::getSubpath)); - DUResponse.DiskUsage duVol1 = duRootData.get(0); - DUResponse.DiskUsage duVol2 = duRootData.get(1); - assertEquals(VOL_PATH, duVol1.getSubpath()); - assertEquals(VOL_TWO_PATH, duVol2.getSubpath()); - assertEquals(VOL_DATA_SIZE, duVol1.getSize()); - assertEquals(VOL_TWO_DATA_SIZE, duVol2.getSize()); - } - - @Test - public void testDiskUsageVolume() throws Exception { - // volume level DU - Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_PATH, - false, false, false); - DUResponse duVolRes = (DUResponse) volResponse.getEntity(); - assertEquals(2, duVolRes.getCount()); - List duData = duVolRes.getDuData(); - // sort based on subpath - Collections.sort(duData, - Comparator.comparing(DUResponse.DiskUsage::getSubpath)); - DUResponse.DiskUsage duBucket1 = duData.get(0); - DUResponse.DiskUsage duBucket2 = duData.get(1); - assertEquals(BUCKET_ONE_PATH, duBucket1.getSubpath()); - assertEquals(BUCKET_TWO_PATH, duBucket2.getSubpath()); - assertEquals(BUCKET_ONE_DATA_SIZE, duBucket1.getSize()); - assertEquals(BUCKET_TWO_DATA_SIZE, duBucket2.getSize()); - } - - @Test - public void testDiskUsageVolTwo() throws Exception { - // volume level DU - Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_TWO_PATH, - false, false, false); - DUResponse duVolRes = (DUResponse) volResponse.getEntity(); - assertEquals(2, duVolRes.getCount()); - List duData = duVolRes.getDuData(); - // sort based on subpath - Collections.sort(duData, - Comparator.comparing(DUResponse.DiskUsage::getSubpath)); - DUResponse.DiskUsage duBucket3 = duData.get(0); - DUResponse.DiskUsage duBucket4 = duData.get(1); - assertEquals(BUCKET_THREE_PATH, duBucket3.getSubpath()); - assertEquals(BUCKET_FOUR_PATH, duBucket4.getSubpath()); - assertEquals(VOL_TWO_DATA_SIZE, duVolRes.getSize()); - } - - @Test - public void testDiskUsageBucketOne() throws Exception { - // bucket level DU - Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_ONE_PATH, - false, false, false); - DUResponse duBucketResponse = (DUResponse) bucketResponse.getEntity(); - // There are no sub-paths under this OBS bucket. - assertEquals(0, duBucketResponse.getCount()); - - Response bucketResponseWithSubpath = nsSummaryEndpoint.getDiskUsage( - BUCKET_ONE_PATH, true, false, false); - DUResponse duBucketResponseWithFiles = - (DUResponse) bucketResponseWithSubpath.getEntity(); - assertEquals(3, duBucketResponseWithFiles.getCount()); - - assertEquals(BUCKET_ONE_DATA_SIZE, duBucketResponse.getSize()); - } - - @Test - public void testDiskUsageBucketTwo() throws Exception { - // bucket level DU - Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_TWO_PATH, - false, false, false); - DUResponse duBucketResponse = (DUResponse) bucketResponse.getEntity(); - // There are no sub-paths under this OBS bucket. - assertEquals(0, duBucketResponse.getCount()); - - Response bucketResponseWithSubpath = nsSummaryEndpoint.getDiskUsage( - BUCKET_TWO_PATH, true, false, false); - DUResponse duBucketResponseWithFiles = - (DUResponse) bucketResponseWithSubpath.getEntity(); - assertEquals(2, duBucketResponseWithFiles.getCount()); - - assertEquals(BUCKET_TWO_DATA_SIZE, duBucketResponse.getSize()); - } - - @Test - public void testDiskUsageBucketThree() throws Exception { - // bucket level DU - Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_THREE_PATH, - false, false, false); - DUResponse duBucketResponse = (DUResponse) bucketResponse.getEntity(); - // There are no sub-paths under this Legacy bucket. - assertEquals(0, duBucketResponse.getCount()); - - Response bucketResponseWithSubpath = nsSummaryEndpoint.getDiskUsage( - BUCKET_THREE_PATH, true, false, false); - DUResponse duBucketResponseWithFiles = - (DUResponse) bucketResponseWithSubpath.getEntity(); - assertEquals(3, duBucketResponseWithFiles.getCount()); - - assertEquals(BUCKET_THREE_DATA_SIZE, duBucketResponse.getSize()); - } - - @Test - public void testDiskUsageKey1() throws Exception { - // key level DU - Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_ONE_PATH, - false, false, false); - DUResponse duKeyResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(0, duKeyResponse.getCount()); - assertEquals(FILE_ONE_SIZE, duKeyResponse.getSize()); - } - - @Test - public void testDiskUsageKey2() throws Exception { - // key level DU - Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_TWO_PATH, - false, false, false); - DUResponse duKeyResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(0, duKeyResponse.getCount()); - assertEquals(FILE_TWO_SIZE, duKeyResponse.getSize()); - } - - @Test - public void testDiskUsageKey4() throws Exception { - // key level DU - Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY4_PATH, - true, false, false); - DUResponse duKeyResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(0, duKeyResponse.getCount()); - assertEquals(FILE_FOUR_SIZE, duKeyResponse.getSize()); - } - - @Test - public void testDiskUsageKey5() throws Exception { - // key level DU - Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_FIVE_PATH, - false, false, false); - DUResponse duKeyResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(0, duKeyResponse.getCount()); - assertEquals(FILE_FIVE_SIZE, duKeyResponse.getSize()); - } - - @Test - public void testDiskUsageKey8() throws Exception { - // key level DU - Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_EIGHT_PATH, - false, false, false); - DUResponse duKeyResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(0, duKeyResponse.getCount()); - assertEquals(FILE_EIGHT_SIZE, duKeyResponse.getSize()); - } - - @Test - public void testDiskUsageKey11() throws Exception { - // key level DU - Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY_ELEVEN_PATH, - false, false, false); - DUResponse duKeyResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(0, duKeyResponse.getCount()); - assertEquals(FILE_ELEVEN_SIZE, duKeyResponse.getSize()); - } - - @Test - public void testDiskUsageUnknown() throws Exception { - // invalid path check - Response invalidResponse = nsSummaryEndpoint.getDiskUsage(INVALID_PATH, - false, false, false); - DUResponse invalidObj = (DUResponse) invalidResponse.getEntity(); - assertEquals(ResponseStatus.PATH_NOT_FOUND, - invalidObj.getStatus()); - } - - @Test - public void testDiskUsageWithReplication() throws Exception { - setUpMultiBlockKey(); - Response keyResponse = nsSummaryEndpoint.getDiskUsage(MULTI_BLOCK_KEY_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_KEY_SIZE_WITH_REPLICA, - replicaDUResponse.getSizeWithReplica()); - } - - @Test - public void testDataSizeUnderRootWithReplication() throws IOException { - setUpMultiBlockReplicatedKeys(); - // withReplica is true - Response rootResponse = nsSummaryEndpoint.getDiskUsage(ROOT_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) rootResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_ROOT, - replicaDUResponse.getSizeWithReplica()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_VOL, - replicaDUResponse.getDuData().get(0).getSizeWithReplica()); - - } - - @Test - public void testDataSizeUnderVolWithReplication() throws IOException { - setUpMultiBlockReplicatedKeys(); - Response volResponse = nsSummaryEndpoint.getDiskUsage(VOL_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) volResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_VOL, - replicaDUResponse.getSizeWithReplica()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET1, - replicaDUResponse.getDuData().get(0).getSizeWithReplica()); - } - - @Test - public void testDataSizeUnderBucketOneWithReplication() throws IOException { - setUpMultiBlockReplicatedKeys(); - Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_ONE_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) bucketResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET1, - replicaDUResponse.getSizeWithReplica()); - } - - @Test - public void testDataSizeUnderBucketThreeWithReplication() throws IOException { - setUpMultiBlockReplicatedKeys(); - Response bucketResponse = nsSummaryEndpoint.getDiskUsage(BUCKET_THREE_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) bucketResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_BUCKET3, - replicaDUResponse.getSizeWithReplica()); - } - - @Test - public void testDataSizeUnderKeyWithReplication() throws IOException { - setUpMultiBlockReplicatedKeys(); - Response keyResponse = nsSummaryEndpoint.getDiskUsage(KEY4_PATH, - false, true, false); - DUResponse replicaDUResponse = (DUResponse) keyResponse.getEntity(); - assertEquals(ResponseStatus.OK, replicaDUResponse.getStatus()); - assertEquals(MULTI_BLOCK_TOTAL_SIZE_WITH_REPLICA_UNDER_KEY, - replicaDUResponse.getSizeWithReplica()); - } - - @Test - public void testQuotaUsage() throws Exception { - // root level quota usage - Response rootResponse = nsSummaryEndpoint.getQuotaUsage(ROOT_PATH); - QuotaUsageResponse quRootRes = - (QuotaUsageResponse) rootResponse.getEntity(); - assertEquals(ROOT_QUOTA, quRootRes.getQuota()); - assertEquals(ROOT_DATA_SIZE, quRootRes.getQuotaUsed()); - - // volume level quota usage - Response volResponse = nsSummaryEndpoint.getQuotaUsage(VOL_PATH); - QuotaUsageResponse quVolRes = (QuotaUsageResponse) volResponse.getEntity(); - assertEquals(VOL_QUOTA, quVolRes.getQuota()); - assertEquals(VOL_DATA_SIZE, quVolRes.getQuotaUsed()); - - // bucket level quota usage - Response bucketRes = nsSummaryEndpoint.getQuotaUsage(BUCKET_ONE_PATH); - QuotaUsageResponse quBucketRes = (QuotaUsageResponse) bucketRes.getEntity(); - assertEquals(BUCKET_ONE_QUOTA, quBucketRes.getQuota()); - assertEquals(BUCKET_ONE_DATA_SIZE, quBucketRes.getQuotaUsed()); - - Response bucketRes2 = nsSummaryEndpoint.getQuotaUsage(BUCKET_TWO_PATH); - QuotaUsageResponse quBucketRes2 = - (QuotaUsageResponse) bucketRes2.getEntity(); - assertEquals(BUCKET_TWO_QUOTA, quBucketRes2.getQuota()); - assertEquals(BUCKET_TWO_DATA_SIZE, quBucketRes2.getQuotaUsed()); - - Response bucketRes3 = nsSummaryEndpoint.getQuotaUsage(BUCKET_THREE_PATH); - QuotaUsageResponse quBucketRes3 = - (QuotaUsageResponse) bucketRes3.getEntity(); - assertEquals(BUCKET_THREE_QUOTA, quBucketRes3.getQuota()); - assertEquals(BUCKET_THREE_DATA_SIZE, quBucketRes3.getQuotaUsed()); - - Response bucketRes4 = nsSummaryEndpoint.getQuotaUsage(BUCKET_FOUR_PATH); - QuotaUsageResponse quBucketRes4 = - (QuotaUsageResponse) bucketRes4.getEntity(); - assertEquals(BUCKET_FOUR_QUOTA, quBucketRes4.getQuota()); - assertEquals(BUCKET_FOUR_DATA_SIZE, quBucketRes4.getQuotaUsed()); - - // other level not applicable - Response naResponse2 = nsSummaryEndpoint.getQuotaUsage(KEY4_PATH); - QuotaUsageResponse quotaUsageResponse2 = - (QuotaUsageResponse) naResponse2.getEntity(); - assertEquals(ResponseStatus.TYPE_NOT_APPLICABLE, - quotaUsageResponse2.getResponseCode()); - - // invalid path request - Response invalidRes = nsSummaryEndpoint.getQuotaUsage(INVALID_PATH); - QuotaUsageResponse invalidResObj = - (QuotaUsageResponse) invalidRes.getEntity(); - assertEquals(ResponseStatus.PATH_NOT_FOUND, - invalidResObj.getResponseCode()); - } - - @Test - public void testFileSizeDist() throws Exception { - checkFileSizeDist(ROOT_PATH, 2, 3, 3, 1); - checkFileSizeDist(VOL_PATH, 2, 1, 1, 1); - checkFileSizeDist(BUCKET_ONE_PATH, 1, 1, 0, 1); - } - - public void checkFileSizeDist(String path, int bin0, - int bin1, int bin2, int bin3) throws Exception { - Response res = nsSummaryEndpoint.getFileSizeDistribution(path); - FileSizeDistributionResponse fileSizeDistResObj = - (FileSizeDistributionResponse) res.getEntity(); - int[] fileSizeDist = fileSizeDistResObj.getFileSizeDist(); - assertEquals(bin0, fileSizeDist[0]); - assertEquals(bin1, fileSizeDist[1]); - assertEquals(bin2, fileSizeDist[2]); - assertEquals(bin3, fileSizeDist[3]); - for (int i = 4; i < ReconConstants.NUM_OF_FILE_SIZE_BINS; ++i) { - assertEquals(0, fileSizeDist[i]); - } - } - - @Test - public void testNormalizePathUptoBucket() { - // Test null or empty path - assertEquals("/", OmUtils.normalizePathUptoBucket(null)); - assertEquals("/", OmUtils.normalizePathUptoBucket("")); - - // Test path with leading slashes - assertEquals("volume1/bucket1/key1/key2", - OmUtils.normalizePathUptoBucket("///volume1/bucket1/key1/key2")); - - // Test volume and bucket names - assertEquals("volume1/bucket1", - OmUtils.normalizePathUptoBucket("volume1/bucket1")); - - // Test with additional segments - assertEquals("volume1/bucket1/key1/key2", - OmUtils.normalizePathUptoBucket("volume1/bucket1/key1/key2")); - - // Test path with multiple slashes in key names. - assertEquals("volume1/bucket1/key1//key2", - OmUtils.normalizePathUptoBucket("volume1/bucket1/key1//key2")); - - // Test path with volume, bucket, and special characters in keys - assertEquals("volume/bucket/key$%#1/./////////key$%#2", - OmUtils.normalizePathUptoBucket("volume/bucket/key$%#1/./////////key$%#2")); - } - - @Test - public void testConstructFullPath() throws IOException { - OmKeyInfo keyInfo = new OmKeyInfo.Builder() - .setKeyName(KEY_TWO) - .setVolumeName(VOL) - .setBucketName(BUCKET_ONE) - .setObjectID(KEY_TWO_OBJECT_ID) - .build(); - String fullPath = ReconUtils.constructFullPath(keyInfo, - reconNamespaceSummaryManager); - String expectedPath = "vol/bucket1/" + KEY_TWO; - Assertions.assertEquals(expectedPath, fullPath); - - keyInfo = new OmKeyInfo.Builder() - .setKeyName(KEY_FIVE) - .setVolumeName(VOL) - .setBucketName(BUCKET_TWO) - .setObjectID(KEY_FIVE_OBJECT_ID) - .build(); - fullPath = ReconUtils.constructFullPath(keyInfo, - reconNamespaceSummaryManager); - expectedPath = "vol/bucket2/" + KEY_FIVE; - Assertions.assertEquals(expectedPath, fullPath); - - keyInfo = new OmKeyInfo.Builder() - .setKeyName(KEY_EIGHT) - .setVolumeName(VOL_TWO) - .setBucketName(BUCKET_THREE) - .setObjectID(KEY_EIGHT_OBJECT_ID) - .build(); - fullPath = ReconUtils.constructFullPath(keyInfo, - reconNamespaceSummaryManager); - expectedPath = "vol2/bucket3/" + KEY_EIGHT; - Assertions.assertEquals(expectedPath, fullPath); - - - keyInfo = new OmKeyInfo.Builder() - .setKeyName(KEY_ELEVEN) - .setVolumeName(VOL_TWO) - .setBucketName(BUCKET_FOUR) - .setObjectID(KEY_ELEVEN_OBJECT_ID) - .build(); - fullPath = ReconUtils.constructFullPath(keyInfo, - reconNamespaceSummaryManager); - expectedPath = "vol2/bucket4/" + KEY_ELEVEN; - Assertions.assertEquals(expectedPath, fullPath); - } - - - /** - * Testing the following case. - * └── vol - * ├── bucket1 (OBS) - * │ ├── file1 - * │ ├── file2 - * │ └── file3 - * └── bucket2 (OBS) - * ├── file4 - * └── file5 - * └── vol2 - * ├── bucket3 (Legacy) - * │ ├── file8 - * │ ├── file9 - * │ └── file10 - * └── bucket4 (Legacy) - * └── file11 - * - * Write these keys to OM and - * replicate them. - * @throws Exception - */ - @SuppressWarnings("checkstyle:MethodLength") - private void populateOMDB() throws Exception { - - // write all keys - writeKeyToOm(reconOMMetadataManager, - KEY_ONE, - BUCKET_ONE, - VOL, - KEY_ONE, - KEY_ONE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - FILE_ONE_SIZE, - getOBSBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_TWO, - BUCKET_ONE, - VOL, - KEY_TWO, - KEY_TWO_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - FILE_TWO_SIZE, - getOBSBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_THREE, - BUCKET_ONE, - VOL, - KEY_THREE, - KEY_THREE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - FILE_THREE_SIZE, - getOBSBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_FOUR, - BUCKET_TWO, - VOL, - KEY_FOUR, - KEY_FOUR_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - VOL_OBJECT_ID, - FILE_FOUR_SIZE, - getOBSBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_FIVE, - BUCKET_TWO, - VOL, - KEY_FIVE, - KEY_FIVE_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - VOL_OBJECT_ID, - FILE_FIVE_SIZE, - getOBSBucketLayout()); - - writeKeyToOm(reconOMMetadataManager, - KEY_EIGHT, - BUCKET_THREE, - VOL_TWO, - KEY_EIGHT, - KEY_EIGHT_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - FILE_EIGHT_SIZE, - getLegacyBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_NINE, - BUCKET_THREE, - VOL_TWO, - KEY_NINE, - KEY_NINE_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - FILE_NINE_SIZE, - getLegacyBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_TEN, - BUCKET_THREE, - VOL_TWO, - KEY_TEN, - KEY_TEN_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - FILE_TEN_SIZE, - getLegacyBucketLayout()); - writeKeyToOm(reconOMMetadataManager, - KEY_ELEVEN, - BUCKET_FOUR, - VOL_TWO, - KEY_ELEVEN, - KEY_ELEVEN_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_FOUR_OBJECT_ID, - VOL_TWO_OBJECT_ID, - FILE_ELEVEN_SIZE, - getLegacyBucketLayout()); - } - - /** - * Create a new OM Metadata manager instance with one user, one vol, and two - * buckets. - * - * @throws IOException ioEx - */ - private static OMMetadataManager initializeNewOmMetadataManager( - File omDbDir, OzoneConfiguration omConfiguration) - throws IOException { - omConfiguration.set(OZONE_OM_DB_DIRS, - omDbDir.getAbsolutePath()); - omConfiguration.set(OMConfigKeys - .OZONE_OM_ENABLE_FILESYSTEM_PATHS, "false"); - OMMetadataManager omMetadataManager = new OmMetadataManagerImpl( - omConfiguration, null); - - String volumeKey = omMetadataManager.getVolumeKey(VOL); - OmVolumeArgs args = - OmVolumeArgs.newBuilder() - .setObjectID(VOL_OBJECT_ID) - .setVolume(VOL) - .setAdminName(TEST_USER) - .setOwnerName(TEST_USER) - .setQuotaInBytes(VOL_QUOTA) - .build(); - - String volume2Key = omMetadataManager.getVolumeKey(VOL_TWO); - OmVolumeArgs args2 = - OmVolumeArgs.newBuilder() - .setObjectID(VOL_TWO_OBJECT_ID) - .setVolume(VOL_TWO) - .setAdminName(TEST_USER) - .setOwnerName(TEST_USER) - .setQuotaInBytes(VOL_TWO_QUOTA) - .build(); - - omMetadataManager.getVolumeTable().put(volumeKey, args); - omMetadataManager.getVolumeTable().put(volume2Key, args2); - - OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() - .setVolumeName(VOL) - .setBucketName(BUCKET_ONE) - .setObjectID(BUCKET_ONE_OBJECT_ID) - .setQuotaInBytes(BUCKET_ONE_QUOTA) - .setBucketLayout(getOBSBucketLayout()) - .build(); - - OmBucketInfo bucketInfo2 = OmBucketInfo.newBuilder() - .setVolumeName(VOL) - .setBucketName(BUCKET_TWO) - .setObjectID(BUCKET_TWO_OBJECT_ID) - .setQuotaInBytes(BUCKET_TWO_QUOTA) - .setBucketLayout(getOBSBucketLayout()) - .build(); - - OmBucketInfo bucketInfo3 = OmBucketInfo.newBuilder() - .setVolumeName(VOL_TWO) - .setBucketName(BUCKET_THREE) - .setObjectID(BUCKET_THREE_OBJECT_ID) - .setQuotaInBytes(BUCKET_THREE_QUOTA) - .setBucketLayout(getLegacyBucketLayout()) - .build(); - - OmBucketInfo bucketInfo4 = OmBucketInfo.newBuilder() - .setVolumeName(VOL_TWO) - .setBucketName(BUCKET_FOUR) - .setObjectID(BUCKET_FOUR_OBJECT_ID) - .setQuotaInBytes(BUCKET_FOUR_QUOTA) - .setBucketLayout(getLegacyBucketLayout()) - .build(); - - String bucketKey = omMetadataManager.getBucketKey( - bucketInfo.getVolumeName(), bucketInfo.getBucketName()); - String bucketKey2 = omMetadataManager.getBucketKey( - bucketInfo2.getVolumeName(), bucketInfo2.getBucketName()); - String bucketKey3 = omMetadataManager.getBucketKey( - bucketInfo3.getVolumeName(), bucketInfo3.getBucketName()); - String bucketKey4 = omMetadataManager.getBucketKey( - bucketInfo4.getVolumeName(), bucketInfo4.getBucketName()); - - omMetadataManager.getBucketTable().put(bucketKey, bucketInfo); - omMetadataManager.getBucketTable().put(bucketKey2, bucketInfo2); - omMetadataManager.getBucketTable().put(bucketKey3, bucketInfo3); - omMetadataManager.getBucketTable().put(bucketKey4, bucketInfo4); - - return omMetadataManager; - } - - private void setUpMultiBlockKey() throws IOException { - OmKeyLocationInfoGroup locationInfoGroup = - getLocationInfoGroup1(); - - // add the multi-block key to Recon's OM - writeKeyToOm(reconOMMetadataManager, - MULTI_BLOCK_FILE, - BUCKET_ONE, - VOL, - MULTI_BLOCK_FILE, - MULTI_BLOCK_KEY_OBJECT_ID, - PARENT_OBJECT_ID_ZERO, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup), - getOBSBucketLayout(), - FILE_THREE_SIZE); - } - - private OmKeyLocationInfoGroup getLocationInfoGroup1() { - List locationInfoList = new ArrayList<>(); - BlockID block1 = new BlockID(CONTAINER_ONE_ID, 0L); - BlockID block2 = new BlockID(CONTAINER_TWO_ID, 0L); - BlockID block3 = new BlockID(CONTAINER_THREE_ID, 0L); - - OmKeyLocationInfo location1 = new OmKeyLocationInfo.Builder() - .setBlockID(block1) - .setLength(BLOCK_ONE_LENGTH) - .build(); - OmKeyLocationInfo location2 = new OmKeyLocationInfo.Builder() - .setBlockID(block2) - .setLength(BLOCK_TWO_LENGTH) - .build(); - OmKeyLocationInfo location3 = new OmKeyLocationInfo.Builder() - .setBlockID(block3) - .setLength(BLOCK_THREE_LENGTH) - .build(); - locationInfoList.add(location1); - locationInfoList.add(location2); - locationInfoList.add(location3); - - return new OmKeyLocationInfoGroup(0L, locationInfoList); - } - - private OmKeyLocationInfoGroup getLocationInfoGroup2() { - List locationInfoList = new ArrayList<>(); - BlockID block4 = new BlockID(CONTAINER_FOUR_ID, 0L); - BlockID block5 = new BlockID(CONTAINER_FIVE_ID, 0L); - BlockID block6 = new BlockID(CONTAINER_SIX_ID, 0L); - - OmKeyLocationInfo location4 = new OmKeyLocationInfo.Builder() - .setBlockID(block4) - .setLength(BLOCK_FOUR_LENGTH) - .build(); - OmKeyLocationInfo location5 = new OmKeyLocationInfo.Builder() - .setBlockID(block5) - .setLength(BLOCK_FIVE_LENGTH) - .build(); - OmKeyLocationInfo location6 = new OmKeyLocationInfo.Builder() - .setBlockID(block6) - .setLength(BLOCK_SIX_LENGTH) - .build(); - locationInfoList.add(location4); - locationInfoList.add(location5); - locationInfoList.add(location6); - return new OmKeyLocationInfoGroup(0L, locationInfoList); - - } - - @SuppressWarnings("checkstyle:MethodLength") - private void setUpMultiBlockReplicatedKeys() throws IOException { - OmKeyLocationInfoGroup locationInfoGroup1 = - getLocationInfoGroup1(); - OmKeyLocationInfoGroup locationInfoGroup2 = - getLocationInfoGroup2(); - - //vol/bucket1/file1 - writeKeyToOm(reconOMMetadataManager, - KEY_ONE, - BUCKET_ONE, - VOL, - KEY_ONE, - KEY_ONE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getOBSBucketLayout(), - FILE_ONE_SIZE); - - //vol/bucket1/file2 - writeKeyToOm(reconOMMetadataManager, - KEY_TWO, - BUCKET_ONE, - VOL, - KEY_TWO, - KEY_TWO_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup2), - getOBSBucketLayout(), - FILE_TWO_SIZE); - - //vol/bucket1/file3 - writeKeyToOm(reconOMMetadataManager, - KEY_THREE, - BUCKET_ONE, - VOL, - KEY_THREE, - KEY_THREE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - BUCKET_ONE_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getOBSBucketLayout(), - FILE_THREE_SIZE); - - //vol/bucket2/file4 - writeKeyToOm(reconOMMetadataManager, - KEY_FOUR, - BUCKET_TWO, - VOL, - KEY_FOUR, - KEY_FOUR_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup2), - getOBSBucketLayout(), - FILE_FOUR_SIZE); - - //vol/bucket2/file5 - writeKeyToOm(reconOMMetadataManager, - KEY_FIVE, - BUCKET_TWO, - VOL, - KEY_FIVE, - KEY_FIVE_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - BUCKET_TWO_OBJECT_ID, - VOL_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getOBSBucketLayout(), - FILE_FIVE_SIZE); - - //vol2/bucket3/file8 - writeKeyToOm(reconOMMetadataManager, - KEY_EIGHT, - BUCKET_THREE, - VOL_TWO, - KEY_EIGHT, - KEY_EIGHT_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - Collections.singletonList(locationInfoGroup2), - getLegacyBucketLayout(), - FILE_EIGHT_SIZE); - - //vol2/bucket3/file9 - writeKeyToOm(reconOMMetadataManager, - KEY_NINE, - BUCKET_THREE, - VOL_TWO, - KEY_NINE, - KEY_NINE_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getLegacyBucketLayout(), - FILE_NINE_SIZE); - - //vol2/bucket3/file10 - writeKeyToOm(reconOMMetadataManager, - KEY_TEN, - BUCKET_THREE, - VOL_TWO, - KEY_TEN, - KEY_TEN_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - BUCKET_THREE_OBJECT_ID, - VOL_TWO_OBJECT_ID, - Collections.singletonList(locationInfoGroup2), - getLegacyBucketLayout(), - FILE_TEN_SIZE); - - //vol2/bucket4/file11 - writeKeyToOm(reconOMMetadataManager, - KEY_ELEVEN, - BUCKET_FOUR, - VOL_TWO, - KEY_ELEVEN, - KEY_ELEVEN_OBJECT_ID, - BUCKET_FOUR_OBJECT_ID, - BUCKET_FOUR_OBJECT_ID, - VOL_TWO_OBJECT_ID, - Collections.singletonList(locationInfoGroup1), - getLegacyBucketLayout(), - FILE_ELEVEN_SIZE); - } - - /** - * Generate a set of mock container replica with a size of - * replication factor for container. - * - * @param replicationFactor number of replica - * @param containerID the container replicated based upon - * @return a set of container replica for testing - */ - private static Set generateMockContainerReplicas( - int replicationFactor, ContainerID containerID) { - Set result = new HashSet<>(); - for (int i = 0; i < replicationFactor; ++i) { - DatanodeDetails randomDatanode = randomDatanodeDetails(); - ContainerReplica replica = new ContainerReplica.ContainerReplicaBuilder() - .setContainerID(containerID) - .setContainerState( - StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.OPEN) - .setDatanodeDetails(randomDatanode) - .build(); - result.add(replica); - } - return result; - } - - private static ReconStorageContainerManagerFacade getMockReconSCM() - throws ContainerNotFoundException { - ReconStorageContainerManagerFacade reconSCM = - mock(ReconStorageContainerManagerFacade.class); - ContainerManager containerManager = mock(ContainerManager.class); - - // Container 1 is 3-way replicated - ContainerID containerID1 = ContainerID.valueOf(CONTAINER_ONE_ID); - Set containerReplicas1 = generateMockContainerReplicas( - CONTAINER_ONE_REPLICA_COUNT, containerID1); - when(containerManager.getContainerReplicas(containerID1)) - .thenReturn(containerReplicas1); - - // Container 2 is under replicated with 2 replica - ContainerID containerID2 = ContainerID.valueOf(CONTAINER_TWO_ID); - Set containerReplicas2 = generateMockContainerReplicas( - CONTAINER_TWO_REPLICA_COUNT, containerID2); - when(containerManager.getContainerReplicas(containerID2)) - .thenReturn(containerReplicas2); - - // Container 3 is over replicated with 4 replica - ContainerID containerID3 = ContainerID.valueOf(CONTAINER_THREE_ID); - Set containerReplicas3 = generateMockContainerReplicas( - CONTAINER_THREE_REPLICA_COUNT, containerID3); - when(containerManager.getContainerReplicas(containerID3)) - .thenReturn(containerReplicas3); - - // Container 4 is replicated with 5 replica - ContainerID containerID4 = ContainerID.valueOf(CONTAINER_FOUR_ID); - Set containerReplicas4 = generateMockContainerReplicas( - CONTAINER_FOUR_REPLICA_COUNT, containerID4); - when(containerManager.getContainerReplicas(containerID4)) - .thenReturn(containerReplicas4); - - // Container 5 is replicated with 2 replica - ContainerID containerID5 = ContainerID.valueOf(CONTAINER_FIVE_ID); - Set containerReplicas5 = generateMockContainerReplicas( - CONTAINER_FIVE_REPLICA_COUNT, containerID5); - when(containerManager.getContainerReplicas(containerID5)) - .thenReturn(containerReplicas5); - - // Container 6 is replicated with 3 replica - ContainerID containerID6 = ContainerID.valueOf(CONTAINER_SIX_ID); - Set containerReplicas6 = generateMockContainerReplicas( - CONTAINER_SIX_REPLICA_COUNT, containerID6); - when(containerManager.getContainerReplicas(containerID6)) - .thenReturn(containerReplicas6); - - when(reconSCM.getContainerManager()).thenReturn(containerManager); - ReconNodeManager mockReconNodeManager = mock(ReconNodeManager.class); - when(mockReconNodeManager.getStats()).thenReturn(getMockSCMRootStat()); - when(reconSCM.getScmNodeManager()).thenReturn(mockReconNodeManager); - return reconSCM; - } - - private static BucketLayout getOBSBucketLayout() { - return BucketLayout.OBJECT_STORE; - } - - private static BucketLayout getLegacyBucketLayout() { - return BucketLayout.LEGACY; - } - - private static SCMNodeStat getMockSCMRootStat() { - return new SCMNodeStat(ROOT_QUOTA, ROOT_DATA_SIZE, - ROOT_QUOTA - ROOT_DATA_SIZE, 0L, 0L, 0); - } - -}