Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
32609b8
feat: Strategy Interface Definition and Implementation
suni72 Jun 10, 2026
daaeb1a
feat: File System Integration and Routing Mechanics
suni72 Jun 10, 2026
e53dea2
refactor: fix import paths and rmeove redundant gcsFilesystemOptions
suni72 Jun 16, 2026
1d36799
refactor: remove unused filesystem operations from GcsFileSystem
suni72 Jun 17, 2026
1ca0bed
test: add HNS support validation tests
suni72 Jun 17, 2026
b4f8daa
refactor: encapsulate namespace strategy accessors
suni72 Jun 17, 2026
d7ccf9c
refactor: migrate BucketCapabilities to an AutoValue class
suni72 Jun 17, 2026
7890c16
refactor: replace BucketCapabilities with BucketProperties
suni72 Jun 30, 2026
58c3cf1
refactor: Move NamespaceStrategy to client package and functionally i…
suni72 Jul 7, 2026
c5316b6
doc: add HNS API configuration in doc and annotate resolveStrategy wi…
suni72 Jul 7, 2026
8903dd0
refactor: inject GcsClient into namespace strategies and improve code…
suni72 Jul 14, 2026
55a147a
feat: optimize namespace strategy resolution and add javadoc for name…
suni72 Jul 14, 2026
62864bd
Merge branch 'main' into dir-metadata-ops-2
suni72 Jul 28, 2026
6a44024
refactor: remove BucketPropertiesLoader dependency in favor of GcsCli…
suni72 Jul 29, 2026
5d37642
refactor: update NamespaceStrategy interface methods for improved dir…
suni72 Jul 29, 2026
04c50da
Merge branch 'main' into dir-metadata-ops-2
suni72 Jul 29, 2026
bf608db
test: add unit tests for GcsClientImpl.isHnsBucket method
suni72 Jul 29, 2026
b2185df
Merge branch 'main' into dir-metadata-ops-2
suni72 Aug 3, 2026
102fbfd
address reviewer comments:
suni72 Aug 3, 2026
b2a3c6f
test: refactor GcsFileSystemImplTest to improve test readability and …
suni72 Aug 5, 2026
21def19
Merge branch 'main' into dir-metadata-ops-2
suni72 Aug 5, 2026
5e53b88
fix: wrap IOException in UncheckedIOException during bucket property …
suni72 Aug 5, 2026
fa451d6
feat: enable HNS API by default in GcsFileSystemOptions
suni72 Aug 5, 2026
6e2ea01
Merge branch 'main' into dir-metadata-ops-2
suni72 Aug 6, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion CONFIGURATION.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,6 @@ These parameters fine-tune the low-level data streaming behavior. They allow you
| `analytics-core.adaptive-read.sequential-read-threshold` | Threshold for number of sequential reads to switch to sequential mode. | `3` |
| `analytics-core.random-read.min-request-size` | Minimum request size for random reads. If the requested read size is smaller, it reads up to this size. | `131072` (128 KB) |


### Telemetry and Monitoring

These settings enable the emission of deep internal metrics—such as cache hit rates, operational durations, and throughput—to local logging consoles or distributed OpenTelemetry backends like Google Cloud Monitoring.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
/*
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.google.cloud.gcs.analyticscore.client;

final class FlatNamespaceStrategyImpl implements NamespaceStrategy {
private final GcsClient gcsClient;

FlatNamespaceStrategyImpl(GcsClient gcsClient) {
this.gcsClient = gcsClient;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ WritableByteChannel createWriteChannel(GcsItemId itemId, GcsWriteOptions options
/** Fetches object metadata. */
GcsItemInfo getGcsItemInfo(GcsItemId itemId) throws IOException;

boolean isHnsBucket(String bucketName) throws IOException;

/** Close the client. */
void close();
}
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,11 @@ BucketProperties getBucketProperties(String bucketName) throws IOException {
}
}

@Override
public boolean isHnsBucket(String bucketName) throws IOException {
return getBucketProperties(bucketName).isHnsEnabled();
}

@Override
public void close() {
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import com.google.common.collect.ImmutableList;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.URI;
import java.nio.channels.WritableByteChannel;
import java.util.Collections;
Expand All @@ -51,6 +52,9 @@ public class GcsFileSystemImpl implements GcsFileSystem {
private final Telemetry telemetry;
private final AnalyticsCacheManager cacheManager;

private final FlatNamespaceStrategyImpl flatStrategy;
private final HierarchicalNamespaceStrategyImpl hnsStrategy;

public GcsFileSystemImpl(GcsFileSystemOptions fileSystemOptions) {
this.fileSystemOptions = fileSystemOptions;
this.executorServiceSupplier = initializeExecutionServiceSupplier();
Expand All @@ -64,6 +68,8 @@ public GcsFileSystemImpl(GcsFileSystemOptions fileSystemOptions) {
recorder ->
new GcsClientImpl(
fileSystemOptions.getGcsClientOptions(), executorServiceSupplier, telemetry));
this.flatStrategy = new FlatNamespaceStrategyImpl(this.gcsClient);
this.hnsStrategy = new HierarchicalNamespaceStrategyImpl(this.gcsClient);
}

public GcsFileSystemImpl(Credentials credentials, GcsFileSystemOptions fileSystemOptions) {
Expand All @@ -82,6 +88,8 @@ public GcsFileSystemImpl(Credentials credentials, GcsFileSystemOptions fileSyste
fileSystemOptions.getGcsClientOptions(),
executorServiceSupplier,
telemetry));
this.flatStrategy = new FlatNamespaceStrategyImpl(this.gcsClient);
this.hnsStrategy = new HierarchicalNamespaceStrategyImpl(this.gcsClient);
}

@VisibleForTesting
Expand All @@ -104,6 +112,32 @@ public GcsFileSystemImpl(Credentials credentials, GcsFileSystemOptions fileSyste
this.executorServiceSupplier = initializeExecutionServiceSupplier();
this.telemetry = telemetry;
this.cacheManager = cacheManager;
this.flatStrategy = new FlatNamespaceStrategyImpl(this.gcsClient);
this.hnsStrategy = new HierarchicalNamespaceStrategyImpl(this.gcsClient);
}

@VisibleForTesting
NamespaceStrategy resolveStrategy(String bucketName) throws IOException {
checkNotNull(bucketName, "bucketName cannot be null");
if (!fileSystemOptions.isHnsApiEnabled()) {
return flatStrategy;
}

BucketProperties properties =
cacheManager.getBucketProperties(
bucketName,
name -> {
try {
return BucketProperties.create(gcsClient.isHnsBucket(name));
} catch (IOException e) {
throw new UncheckedIOException(e);
}
});

if (properties.isHnsEnabled()) {
return hnsStrategy;
}
return flatStrategy;
}

@Override
Expand Down Expand Up @@ -163,6 +197,16 @@ public AnalyticsCacheManager getCacheManager() {
return cacheManager;
}

@VisibleForTesting
FlatNamespaceStrategyImpl getFlatStrategy() {
return flatStrategy;
}

@VisibleForTesting
HierarchicalNamespaceStrategyImpl getHnsStrategy() {
return hnsStrategy;
}

Comment thread
suni72 marked this conversation as resolved.
@Override
public void close() {
ExecutorService executorService = executorServiceSupplier.get();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ public abstract class GcsFileSystemOptions {

private static final String READ_THREAD_COUNT_KEY = "analytics-core.read.thread.count";
private static final String CLIENT_TYPE_KEY = "client.type";
private static final String HNS_API_ENABLED_KEY = "analytics-core.hierarchical.namespace.enable";

/** Cloud Storage client to use. */
public enum ClientType {
Expand All @@ -43,12 +44,15 @@ public enum ClientType {

public abstract TelemetryOptions getAnalyticsCoreTelemetryOptions();

public abstract boolean isHnsApiEnabled();

public abstract Builder toBuilder();

public static Builder builder() {
return new AutoValue_GcsFileSystemOptions.Builder()
.setReadThreadCount(16)
.setClientType(ClientType.HTTP_CLIENT)
.setHnsApiEnabled(true)
.setGcsClientOptions(GcsClientOptions.builder().build())
.setGcsCacheOptions(GcsCacheOptions.builder().build())
.setAnalyticsCoreTelemetryOptions(TelemetryOptions.builder().build());
Expand All @@ -65,6 +69,11 @@ public static GcsFileSystemOptions createFromOptions(
optionsBuilder.setClientType(
ClientType.valueOf(analyticsCoreOptions.get(prefix + CLIENT_TYPE_KEY)));
}
if (analyticsCoreOptions.containsKey(prefix + HNS_API_ENABLED_KEY)) {
optionsBuilder.setHnsApiEnabled(
Boolean.parseBoolean(analyticsCoreOptions.get(prefix + HNS_API_ENABLED_KEY)));
}

optionsBuilder.setGcsClientOptions(
GcsClientOptions.createFromOptions(analyticsCoreOptions, prefix));
optionsBuilder.setGcsCacheOptions(
Expand All @@ -84,6 +93,8 @@ public abstract static class Builder {

public abstract Builder setReadThreadCount(int readThreadCount);

public abstract Builder setHnsApiEnabled(boolean isHnsApiEnabled);

public abstract Builder setGcsClientOptions(GcsClientOptions gcsClientOptions);

/** Sets the configuration options for the GCS caching layer. */
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
/*
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.google.cloud.gcs.analyticscore.client;

final class HierarchicalNamespaceStrategyImpl implements NamespaceStrategy {
private final GcsClient gcsClient;

HierarchicalNamespaceStrategyImpl(GcsClient gcsClient) {
this.gcsClient = gcsClient;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
/*
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.google.cloud.gcs.analyticscore.client;

/**
* Strategy interface for handling directory operations across different namespace models (Flat vs.
* HNS).
*
* <p>Methods for directory operations will be added in follow-up PRs. These methods will include:
*
* <ul>
* <li>{@code GcsItemInfo getFileInfo(GcsItemId id, PathType pathType) throws IOException;}
* <li>{@code void createDirectory(GcsItemId id) throws IOException;}
* <li>{@code boolean isDirectoryEmpty(GcsItemId id) throws IOException;}
* <li>{@code void renameDirectory(GcsItemId src, GcsItemId dst) throws IOException;}
* <li>{@code java.util.List<GcsItemInfo> listObjectInfo(GcsItemId id) throws IOException;}
* <li>{@code java.util.List<GcsItemInfo> listRecursive(GcsItemId id) throws IOException;}
* </ul>
*/
interface NamespaceStrategy {}
Comment thread
dheerajsngh marked this conversation as resolved.
Original file line number Diff line number Diff line change
Expand Up @@ -83,8 +83,10 @@ class GcsClientImplTest {
private static final String TEST_OBJECT_ID = "test-object-id";
private static final String TEST_WRITE_OBJECT = "test-write-object";
private static final String TEST_NULL_OPTIONS_OBJECT = "test-null-options";
private static final String TEST_NON_EXISTENT_OBJECT = "non-existent";
private static final String NON_EXISTENT_BUCKET = "non-existent-bucket";
private static final String TEST_NON_EXISTENT_OBJECT = "non-existent-object";
private static final String TEST_HNS_BUCKET = "hns-bucket";
private static final String TEST_FLAT_BUCKET = "flat-bucket";
private static final String TEST_NON_EXISTENT_BUCKET = "non-existent-bucket";
private static final String TEST_OBJECT_NAME = "test-object-name";
private static final String BLOB_WRITE_SESSION_CONFIG_FIELD = "blobWriteSessionConfig";
private static final int MB = 1024 * 1024;
Expand Down Expand Up @@ -310,39 +312,61 @@ void getBucketProperties_nullBucketName_throwsNullPointerException() {
}

@Test
void getBucketProperties_hnsEnabled_returnsTrue() throws IOException {
void getBucketProperties_hnsBucket_returnsTrue() throws IOException {
Storage mockStorage = mock(Storage.class);
GcsClientImpl localGcsClient = createClientWithMockStorage(mockStorage);
Bucket mockBucket = mockBucketWithHns(true);
doReturn(mockBucket).when(mockStorage).get(eq("hns-bucket"), any(BucketGetOption.class));
doReturn(mockBucket).when(mockStorage).get(eq(TEST_HNS_BUCKET), any(BucketGetOption.class));

BucketProperties bucketProperties = localGcsClient.getBucketProperties("hns-bucket");
BucketProperties bucketProperties = localGcsClient.getBucketProperties(TEST_HNS_BUCKET);

assertThat(bucketProperties.isHnsEnabled()).isTrue();
}

@Test
void getBucketProperties_hnsDisabled_returnsFalse() throws IOException {
void isHnsBucket_hnsBucket_returnsTrue() throws IOException {
Storage mockStorage = mock(Storage.class);
GcsClientImpl localGcsClient = createClientWithMockStorage(mockStorage);
Bucket mockBucket = mockBucketWithHns(true);
doReturn(mockBucket).when(mockStorage).get(eq(TEST_HNS_BUCKET), any(BucketGetOption.class));

boolean isHns = localGcsClient.isHnsBucket(TEST_HNS_BUCKET);

assertThat(isHns).isTrue();
}

@Test
void getBucketProperties_flatBucket_returnsFalse() throws IOException {
Storage mockStorage = mock(Storage.class);
GcsClientImpl localGcsClient = createClientWithMockStorage(mockStorage);
Bucket mockBucket = mockBucketWithHns(false);
doReturn(mockBucket).when(mockStorage).get(eq("flat-bucket"), any(BucketGetOption.class));
doReturn(mockBucket).when(mockStorage).get(eq(TEST_FLAT_BUCKET), any(BucketGetOption.class));

BucketProperties bucketProperties = localGcsClient.getBucketProperties("flat-bucket");
BucketProperties bucketProperties = localGcsClient.getBucketProperties(TEST_FLAT_BUCKET);

assertThat(bucketProperties.isHnsEnabled()).isFalse();
}

@Test
void getBucketProperties_hnsNull_returnsFalse() throws IOException {
void isHnsBucket_flatBucket_returnsFalse() throws IOException {
Storage mockStorage = mock(Storage.class);
GcsClientImpl localGcsClient = createClientWithMockStorage(mockStorage);
Bucket mockBucket = mockBucketWithHns(false);
doReturn(mockBucket).when(mockStorage).get(eq(TEST_FLAT_BUCKET), any(BucketGetOption.class));

boolean isHns = localGcsClient.isHnsBucket(TEST_FLAT_BUCKET);

assertThat(isHns).isFalse();
}

@Test
void getBucketProperties_missingHnsProperty_returnsFalse() throws IOException {
Storage mockStorage = mock(Storage.class);
GcsClientImpl localGcsClient = createClientWithMockStorage(mockStorage);
Bucket mockBucket = mockBucketWithHns(null);
doReturn(mockBucket)
.when(mockStorage)
.get(eq("flat-bucket-null-hns"), any(BucketGetOption.class));
doReturn(mockBucket).when(mockStorage).get(eq(TEST_BUCKET), any(BucketGetOption.class));

BucketProperties bucketProperties = localGcsClient.getBucketProperties("flat-bucket-null-hns");
BucketProperties bucketProperties = localGcsClient.getBucketProperties(TEST_BUCKET);

assertThat(bucketProperties.isHnsEnabled()).isFalse();
}
Expand All @@ -351,9 +375,9 @@ void getBucketProperties_hnsNull_returnsFalse() throws IOException {
void getBucketProperties_bucketNotFound_returnsDisabledHns() throws Exception {
Storage mockStorage = mock(Storage.class);
GcsClientImpl localGcsClient = createClientWithMockStorage(mockStorage);
doReturn(null).when(mockStorage).get(eq(NON_EXISTENT_BUCKET), any(BucketGetOption.class));
doReturn(null).when(mockStorage).get(eq(TEST_NON_EXISTENT_BUCKET), any(BucketGetOption.class));

BucketProperties properties = localGcsClient.getBucketProperties(NON_EXISTENT_BUCKET);
BucketProperties properties = localGcsClient.getBucketProperties(TEST_NON_EXISTENT_BUCKET);

assertThat(properties.isHnsEnabled()).isFalse();
}
Expand Down Expand Up @@ -505,9 +529,12 @@ void create_whenBucketOrObjectNotFound_throwsFileNotFoundException() throws Exce
Storage mockStorage = mock(Storage.class);
GcsClientImpl clientWithMock = createClientWithMockStorage(mockStorage);
GcsItemId itemId =
GcsItemId.builder().setBucketName(NON_EXISTENT_BUCKET).setObjectName(TEST_OBJECT).build();
GcsItemId.builder()
.setBucketName(TEST_NON_EXISTENT_BUCKET)
.setObjectName(TEST_NON_EXISTENT_OBJECT)
.build();
BlobInfo blobInfo =
BlobInfo.newBuilder(BlobId.of(NON_EXISTENT_BUCKET, TEST_OBJECT))
BlobInfo.newBuilder(BlobId.of(TEST_NON_EXISTENT_BUCKET, TEST_NON_EXISTENT_OBJECT))
.setContentType("application/octet-stream")
.build();
StorageException e404 = new StorageException(404, "Not Found");
Expand Down
Loading