From 09b7638463e832ab24ddfe2edefeaef8b72cd21a Mon Sep 17 00:00:00 2001 From: SaketaChalamchala Date: Fri, 10 Jul 2026 15:30:47 -0700 Subject: [PATCH] HDDS-15624. Resolve link bucket properties in listBucket. (cherry picked from commit 228f523de68218907d4658aae0df59a1321a6f35) --- .../apache/hadoop/ozone/om/OzoneManager.java | 93 +++++++++++-------- .../ozone/om/TestBucketManagerImpl.java | 80 ++++++++++++++++ 2 files changed, 132 insertions(+), 41 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 1910d92e9690..3d677165b965 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -2993,8 +2993,13 @@ public List listBuckets(String volumeName, String startKey, volumeName, null, null); } metrics.incNumBucketLists(); - return bucketManager.listBuckets(volumeName, + List buckets = bucketManager.listBuckets(volumeName, startKey, prefix, maxNumOfBuckets, hasSnapshot); + List enrichedBuckets = new ArrayList<>(buckets.size()); + for (OmBucketInfo bucket : buckets) { + enrichedBuckets.add(enrichLinkBucketInfo(bucket)); + } + return enrichedBuckets; } catch (IOException ex) { metrics.incNumBucketListFails(); auditSuccess = false; @@ -3009,6 +3014,50 @@ public List listBuckets(String volumeName, String startKey, } } + /** + * For link buckets, follows the link chain and overlays the source bucket's + * operational properties onto the link's {@link OmBucketInfo}. Non-link + * buckets and dangling links are returned unchanged. + */ + private OmBucketInfo enrichLinkBucketInfo(OmBucketInfo bucketInfo) + throws IOException { + if (!bucketInfo.isLink()) { + return bucketInfo; + } + // We already know that `bucketInfo` is a linked one, + // so we skip one `getBucketInfo` and start with the known link. + ResolvedBucket resolvedBucket = + resolveBucketLink(Pair.of( + bucketInfo.getSourceVolume(), + bucketInfo.getSourceBucket()), + true); + + // If it is a dangling link it means no real bucket exists, + // for example, it could have been deleted, but the links still present. + if (resolvedBucket.isDangling()) { + return bucketInfo; + } + OmBucketInfo realBucket = + bucketManager.getBucketInfo( + resolvedBucket.realVolume(), + resolvedBucket.realBucket()); + // Pass the real bucket metadata in the link bucket info. + return bucketInfo.toBuilder() + .setDefaultReplicationConfig( + realBucket.getDefaultReplicationConfig()) + .setIsVersionEnabled(realBucket.getIsVersionEnabled()) + .setStorageType(realBucket.getStorageType()) + .setQuotaInBytes(realBucket.getQuotaInBytes()) + .setQuotaInNamespace(realBucket.getQuotaInNamespace()) + .setUsedBytes(realBucket.getUsedBytes()) + .setSnapshotUsedBytes(realBucket.getSnapshotUsedBytes()) + .setSnapshotUsedNamespace(realBucket.getSnapshotUsedNamespace()) + .setUsedNamespace(realBucket.getUsedNamespace()) + .addAllMetadata(realBucket.getMetadata()) + .setBucketLayout(realBucket.getBucketLayout()) + .build(); + } + /** * Gets the bucket information. * @@ -3030,46 +3079,8 @@ public OmBucketInfo getBucketInfo(String volume, String bucket) } metrics.incNumBucketInfos(); - OmBucketInfo bucketInfo = bucketManager.getBucketInfo(volume, bucket); - - // No links - return the bucket info right away. - if (!bucketInfo.isLink()) { - return bucketInfo; - } - // Otherwise follow the links to find the real bucket. - // We already know that `bucketInfo` is a linked one, - // so we skip one `getBucketInfo` and start with the known link. - ResolvedBucket resolvedBucket = - resolveBucketLink(Pair.of( - bucketInfo.getSourceVolume(), - bucketInfo.getSourceBucket()), - true); - - // If it is a dangling link it means no real bucket exists, - // for example, it could have been deleted, but the links still present. - if (!resolvedBucket.isDangling()) { - OmBucketInfo realBucket = - bucketManager.getBucketInfo( - resolvedBucket.realVolume(), - resolvedBucket.realBucket()); - // Pass the real bucket metadata in the link bucket info. - return bucketInfo.toBuilder() - .setDefaultReplicationConfig( - realBucket.getDefaultReplicationConfig()) - .setIsVersionEnabled(realBucket.getIsVersionEnabled()) - .setStorageType(realBucket.getStorageType()) - .setQuotaInBytes(realBucket.getQuotaInBytes()) - .setQuotaInNamespace(realBucket.getQuotaInNamespace()) - .setUsedBytes(realBucket.getUsedBytes()) - .setSnapshotUsedBytes(realBucket.getSnapshotUsedBytes()) - .setSnapshotUsedNamespace(realBucket.getSnapshotUsedNamespace()) - .setUsedNamespace(realBucket.getUsedNamespace()) - .addAllMetadata(realBucket.getMetadata()) - .setBucketLayout(realBucket.getBucketLayout()) - .build(); - } - // If no real bucket exists, return the requested one's info. - return bucketInfo; + return enrichLinkBucketInfo( + bucketManager.getBucketInfo(volume, bucket)); } catch (Exception ex) { metrics.incNumBucketInfoFails(); auditSuccess = false; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestBucketManagerImpl.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestBucketManagerImpl.java index 42d748607ac4..1755fb433c42 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestBucketManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestBucketManagerImpl.java @@ -31,10 +31,12 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.List; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.ECReplicationConfig; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.StorageType; @@ -460,4 +462,82 @@ public void testLinkedBucketResolution() throws Exception { bucketInfo.getIsVersionEnabled(), storedLinkBucket.getIsVersionEnabled()); } + + @Test + public void testListBucketsResolvesFsoAndObsLinkLayouts() throws Exception { + String volume = volumeName(); + createSampleVol(volume); + + ECReplicationConfig fsoReplication = new ECReplicationConfig(3, 2); + OmBucketInfo fsoSource = OmBucketInfo.newBuilder() + .setVolumeName(volume) + .setBucketName("fso-source") + .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED) + .setDefaultReplicationConfig(new DefaultReplicationConfig(fsoReplication)) + .build(); + writeClient.createBucket(fsoSource); + + RatisReplicationConfig obsReplication = + RatisReplicationConfig.getInstance(ReplicationFactor.THREE); + OmBucketInfo obsSource = OmBucketInfo.newBuilder() + .setVolumeName(volume) + .setBucketName("obs-source") + .setBucketLayout(BucketLayout.OBJECT_STORE) + .setDefaultReplicationConfig(new DefaultReplicationConfig(obsReplication)) + .build(); + writeClient.createBucket(obsSource); + + OmBucketInfo fsoLink = OmBucketInfo.newBuilder() + .setVolumeName(volume) + .setBucketName("link-fso") + .setSourceVolume(volume) + .setSourceBucket("fso-source") + .build(); + writeClient.createBucket(fsoLink); + + OmBucketInfo obsLink = OmBucketInfo.newBuilder() + .setVolumeName(volume) + .setBucketName("link-obs") + .setSourceVolume(volume) + .setSourceBucket("obs-source") + .build(); + writeClient.createBucket(obsLink); + + OmBucketInfo fsoLinkInfo = writeClient.getBucketInfo(volume, "link-fso"); + assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED, fsoLinkInfo.getBucketLayout()); + assertEquals(fsoReplication, + fsoLinkInfo.getDefaultReplicationConfig().getReplicationConfig()); + + OmBucketInfo obsLinkInfo = writeClient.getBucketInfo(volume, "link-obs"); + assertEquals(BucketLayout.OBJECT_STORE, obsLinkInfo.getBucketLayout()); + assertEquals(obsReplication, + obsLinkInfo.getDefaultReplicationConfig().getReplicationConfig()); + + List listedBuckets = + writeClient.listBuckets(volume, "", "", 100, false); + + OmBucketInfo listedFsoLink = null; + OmBucketInfo listedObsLink = null; + for (OmBucketInfo listedBucket : listedBuckets) { + if ("link-fso".equals(listedBucket.getBucketName())) { + listedFsoLink = listedBucket; + } else if ("link-obs".equals(listedBucket.getBucketName())) { + listedObsLink = listedBucket; + } + } + assertNotNull(listedFsoLink, "link-fso not found in listBuckets response"); + assertNotNull(listedObsLink, "link-obs not found in listBuckets response"); + + assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED, listedFsoLink.getBucketLayout()); + assertEquals(fsoReplication, + listedFsoLink.getDefaultReplicationConfig().getReplicationConfig()); + assertEquals(volume, listedFsoLink.getSourceVolume()); + assertEquals("fso-source", listedFsoLink.getSourceBucket()); + + assertEquals(BucketLayout.OBJECT_STORE, listedObsLink.getBucketLayout()); + assertEquals(obsReplication, + listedObsLink.getDefaultReplicationConfig().getReplicationConfig()); + assertEquals(volume, listedObsLink.getSourceVolume()); + assertEquals("obs-source", listedObsLink.getSourceBucket()); + } }