Skip to content

Commit 9ad0f6d

Browse files
sagnghossagnghos
committed
feat(spanner): Support dynamic TLS certificate and key rotation for Spanner Omni
Add support for zero-downtime dynamic reloading of client certificates/keys (mTLS) and server root CA certificates in Spanner Omni without requiring application or connection pool restarts. Changes: - DynamicKeyManager (com.google.cloud.spanner.omni): An X509ExtendedKeyManager that monitors certificate/key file modification times and lengths on disk. Uses non-blocking tryLock() and atomic versioned alias mapping to reload rotated client certificates and RSA/EC private keys during active TLS handshakes without blocking Netty event loop threads. - DynamicTrustManager (com.google.cloud.spanner.omni): An X509ExtendedTrustManager that dynamically reloads rotated server root CA certificates into an in-memory keystore/trust manager upon file changes. - SpannerOptions & Connection API: - Added Builder.setCaCertificate(String) and getCaCertificate() across SpannerOptions, ConnectionProperties, ConnectionOptions, and SpannerPool. - Updated Builder.useClientCert(String, String) to use dynamic key management. - Preserved raw certificate paths in SpannerOptions and built transient Netty SslContext in prepareBuilder to prevent transport leaks. - SpannerOmniHelper: Added support for spanner.ca_cert_path and updated mTLS setup detection. - Tests: Added comprehensive unit tests covering dynamic certificate/key rotation, CA rotation, multi-CA bundles, fallback on corruption/mismatch, throttling, and options configuration. Fixes b/562755231
1 parent 48e5ce5 commit 9ad0f6d

12 files changed

Lines changed: 1534 additions & 16 deletions

File tree

‎java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java‎

Lines changed: 87 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@
5353
import com.google.cloud.spanner.admin.database.v1.stub.DatabaseAdminStubSettings;
5454
import com.google.cloud.spanner.admin.instance.v1.InstanceAdminSettings;
5555
import com.google.cloud.spanner.admin.instance.v1.stub.InstanceAdminStubSettings;
56+
import com.google.cloud.spanner.omni.DynamicKeyManager;
57+
import com.google.cloud.spanner.omni.DynamicTrustManager;
5658
import com.google.cloud.spanner.omni.SpannerOmniCredentials;
5759
import com.google.cloud.spanner.spi.SpannerRpcFactory;
5860
import com.google.cloud.spanner.spi.v1.ChannelEndpointCacheFactory;
@@ -85,6 +87,7 @@
8587
import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts;
8688
import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder;
8789
import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext;
90+
import io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder;
8891
import io.opencensus.trace.Tracing;
8992
import io.opentelemetry.api.GlobalOpenTelemetry;
9093
import io.opentelemetry.api.OpenTelemetry;
@@ -356,6 +359,9 @@ static GcpChannelPoolOptions mergeWithDefaultChannelPoolOptions(
356359
private final boolean autoTaggingEnabled;
357360
private final List<String> autoTaggingPackages;
358361
private final int autoTaggingTracerLimit;
362+
private final String clientCertificate;
363+
private final String clientCertificateKey;
364+
private final String caCertificate;
359365

360366
enum TracingFramework {
361367
OPEN_CENSUS,
@@ -941,14 +947,21 @@ protected SpannerOptions(Builder builder) {
941947
transportChannelExecutorThreadNameFormat = builder.transportChannelExecutorThreadNameFormat;
942948
channelProvider = builder.channelProvider;
943949
channelEndpointCacheFactory = builder.channelEndpointCacheFactory;
944-
if (builder.mTLSContext != null) {
950+
clientCertificate = builder.clientCertificate;
951+
clientCertificateKey = builder.clientCertificateKey;
952+
caCertificate = builder.caCertificate;
953+
if (builder.omniSslContext != null) {
954+
final SslContext sslContext = builder.omniSslContext;
955+
@SuppressWarnings("rawtypes")
956+
final ApiFunction<ManagedChannelBuilder, ManagedChannelBuilder> parentConfigurator =
957+
builder.channelConfigurator;
945958
channelConfigurator =
946959
channelBuilder -> {
947-
if (builder.channelConfigurator != null) {
948-
channelBuilder = builder.channelConfigurator.apply(channelBuilder);
960+
if (parentConfigurator != null) {
961+
channelBuilder = parentConfigurator.apply(channelBuilder);
949962
}
950963
if (channelBuilder instanceof NettyChannelBuilder) {
951-
((NettyChannelBuilder) channelBuilder).sslContext(builder.mTLSContext);
964+
((NettyChannelBuilder) channelBuilder).sslContext(sslContext);
952965
}
953966
return channelBuilder;
954967
};
@@ -1292,6 +1305,31 @@ public GoogleCredentials getDefaultSpannerOmniCredentials() {
12921305
public static class Builder
12931306
extends ServiceOptions.Builder<Spanner, SpannerOptions, SpannerOptions.Builder> {
12941307
private static Builder prepareBuilder(Builder builder) {
1308+
boolean hasClientCert = !Strings.isNullOrEmpty(builder.clientCertificate);
1309+
boolean hasClientKey = !Strings.isNullOrEmpty(builder.clientCertificateKey);
1310+
boolean hasCaCert = !Strings.isNullOrEmpty(builder.caCertificate);
1311+
1312+
if (hasClientCert || hasClientKey || hasCaCert) {
1313+
if (hasClientCert != hasClientKey) {
1314+
throw new IllegalArgumentException(
1315+
"Both clientCertificate and clientCertificateKey must be provided together");
1316+
}
1317+
try {
1318+
SslContextBuilder sslContextBuilder = GrpcSslContexts.forClient();
1319+
if (hasClientCert) {
1320+
sslContextBuilder.keyManager(
1321+
new DynamicKeyManager(
1322+
new File(builder.clientCertificate), new File(builder.clientCertificateKey)));
1323+
}
1324+
if (hasCaCert) {
1325+
sslContextBuilder.trustManager(
1326+
new DynamicTrustManager(new File(builder.caCertificate)));
1327+
}
1328+
builder.omniSslContext = sslContextBuilder.build();
1329+
} catch (Exception e) {
1330+
throw SpannerExceptionFactory.asSpannerException(e);
1331+
}
1332+
}
12951333
if (builder.instanceType == InstanceType.OMNI) {
12961334
builder.enableBuiltInMetrics = false;
12971335
builder.setProjectId(SPANNER_OMNI_PROJECT_ID);
@@ -1314,7 +1352,7 @@ private static Builder prepareBuilder(Builder builder) {
13141352
}
13151353
if (builder.credentials instanceof SpannerOmniCredentials) {
13161354
((SpannerOmniCredentials) builder.credentials)
1317-
.initChannel(builder.usePlainText, builder.mTLSContext);
1355+
.initChannel(builder.usePlainText, builder.omniSslContext);
13181356
}
13191357
} else {
13201358
if (builder.username != null || builder.secretBytes != null) {
@@ -1399,7 +1437,10 @@ private static Builder prepareBuilder(Builder builder) {
13991437
private MetricsProvider metricsProvider = DefaultMetricsProvider.INSTANCE;
14001438
private boolean enableLocationApi = SpannerOptions.environment.isEnableLocationApi();
14011439
private String monitoringHost = SpannerOptions.environment.getMonitoringHost();
1402-
private SslContext mTLSContext = null;
1440+
private String clientCertificate = null;
1441+
private String clientCertificateKey = null;
1442+
private String caCertificate = null;
1443+
private SslContext omniSslContext = null;
14031444
private boolean usePlainText = false;
14041445
private TransactionOptions defaultTransactionOptions = TransactionOptions.getDefaultInstance();
14051446
private RequestOptions.ClientContext clientContext;
@@ -1517,6 +1558,9 @@ protected Builder() {
15171558
this.autoTaggingEnabled = options.autoTaggingEnabled;
15181559
this.autoTaggingPackages = options.autoTaggingPackages;
15191560
this.autoTaggingTracerLimit = options.autoTaggingTracerLimit;
1561+
this.clientCertificate = options.clientCertificate;
1562+
this.clientCertificateKey = options.clientCertificateKey;
1563+
this.caCertificate = options.caCertificate;
15201564
}
15211565

15221566
@Override
@@ -2240,21 +2284,33 @@ public Builder setEmulatorHost(String emulatorHost) {
22402284

22412285
/**
22422286
* Configures mTLS authentication using the provided client certificate and key files. mTLS via
2243-
* useClientCert is only supported for Spanner Omni instances.
2287+
* useClientCert is only supported for Spanner Omni instances. Certificates and keys are loaded
2288+
* dynamically and reloaded automatically when rotated on disk.
22442289
*
22452290
* @param clientCertificate Path to the client certificate file.
22462291
* @param clientCertificateKey Path to the client private key file.
2247-
* @throws SpannerException If an error occurs while configuring the mTLS context
22482292
*/
22492293
public Builder useClientCert(String clientCertificate, String clientCertificateKey) {
2250-
try {
2251-
this.mTLSContext =
2252-
GrpcSslContexts.forClient()
2253-
.keyManager(new File(clientCertificate), new File(clientCertificateKey))
2254-
.build();
2255-
} catch (Exception e) {
2256-
throw SpannerExceptionFactory.asSpannerException(e);
2257-
}
2294+
Preconditions.checkArgument(
2295+
!Strings.isNullOrEmpty(clientCertificate), "clientCertificate cannot be null or empty");
2296+
Preconditions.checkArgument(
2297+
!Strings.isNullOrEmpty(clientCertificateKey),
2298+
"clientCertificateKey cannot be null or empty");
2299+
this.clientCertificate = clientCertificate;
2300+
this.clientCertificateKey = clientCertificateKey;
2301+
return this;
2302+
}
2303+
2304+
/**
2305+
* Configures the server root CA certificate for SSL/TLS authentication. The CA certificate is
2306+
* loaded dynamically and reloaded automatically when rotated on disk.
2307+
*
2308+
* @param caCertificate Path to the server root CA certificate file.
2309+
*/
2310+
public Builder setCaCertificate(String caCertificate) {
2311+
Preconditions.checkArgument(
2312+
!Strings.isNullOrEmpty(caCertificate), "caCertificate cannot be null or empty");
2313+
this.caCertificate = caCertificate;
22582314
return this;
22592315
}
22602316

@@ -3175,6 +3231,21 @@ protected boolean shouldRefreshRpc(ServiceRpc cachedRpc) {
31753231
return cachedRpc == null || ((SpannerRpc) cachedRpc).isClosed();
31763232
}
31773233

3234+
@Nullable
3235+
public String getClientCertificate() {
3236+
return clientCertificate;
3237+
}
3238+
3239+
@Nullable
3240+
public String getClientCertificateKey() {
3241+
return clientCertificateKey;
3242+
}
3243+
3244+
@Nullable
3245+
public String getCaCertificate() {
3246+
return caCertificate;
3247+
}
3248+
31783249
@SuppressWarnings("unchecked")
31793250
@Override
31803251
public Builder toBuilder() {

‎java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionOptions.java‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import static com.google.cloud.spanner.connection.ConnectionProperties.AUTOCOMMIT;
2020
import static com.google.cloud.spanner.connection.ConnectionProperties.AUTO_CONFIG_EMULATOR;
2121
import static com.google.cloud.spanner.connection.ConnectionProperties.AUTO_PARTITION_MODE;
22+
import static com.google.cloud.spanner.connection.ConnectionProperties.CA_CERTIFICATE;
2223
import static com.google.cloud.spanner.connection.ConnectionProperties.CHANNEL_PROVIDER;
2324
import static com.google.cloud.spanner.connection.ConnectionProperties.CLIENT_CERTIFICATE;
2425
import static com.google.cloud.spanner.connection.ConnectionProperties.CLIENT_KEY;
@@ -168,6 +169,7 @@ public class ConnectionOptions {
168169
static final String DEFAULT_CREDENTIALS = null;
169170
static final String DEFAULT_CLIENT_CERTIFICATE = null;
170171
static final String DEFAULT_CLIENT_KEY = null;
172+
static final String DEFAULT_CA_CERTIFICATE = null;
171173
static final String DEFAULT_OAUTH_TOKEN = null;
172174
static final Integer DEFAULT_MIN_SESSIONS = null;
173175
static final Integer DEFAULT_MAX_SESSIONS = null;
@@ -242,6 +244,9 @@ public class ConnectionOptions {
242244
/** Client key path to establish mTLS */
243245
static final String CLIENT_KEY_PROPERTY_NAME = "clientKey";
244246

247+
/** Server root CA certificate path for SSL/TLS */
248+
static final String CA_CERTIFICATE_PROPERTY_NAME = "caCertificate";
249+
245250
/** Name of the 'autocommit' connection property. */
246251
public static final String AUTOCOMMIT_PROPERTY_NAME = "autocommit";
247252

@@ -676,6 +681,42 @@ public Builder setType(SpannerOptions.InstanceType instanceType) {
676681
return this;
677682
}
678683

684+
/**
685+
* Sets the path to the client certificate file to use for mTLS authentication with Spanner
686+
* Omni.
687+
*
688+
* @param clientCertificate The path to the client certificate file.
689+
* @return this builder
690+
*/
691+
public Builder setClientCertificate(String clientCertificate) {
692+
setConnectionPropertyValue(CLIENT_CERTIFICATE, clientCertificate);
693+
return this;
694+
}
695+
696+
/**
697+
* Sets the path to the client private key file to use for mTLS authentication with Spanner
698+
* Omni.
699+
*
700+
* @param clientCertificateKey The path to the client private key file.
701+
* @return this builder
702+
*/
703+
public Builder setClientCertificateKey(String clientCertificateKey) {
704+
setConnectionPropertyValue(CLIENT_KEY, clientCertificateKey);
705+
return this;
706+
}
707+
708+
/**
709+
* Sets the path to the server root CA certificate file to use for SSL/TLS verification with
710+
* Spanner Omni.
711+
*
712+
* @param caCertificate The path to the root CA certificate file.
713+
* @return this builder
714+
*/
715+
public Builder setCaCertificate(String caCertificate) {
716+
setConnectionPropertyValue(CA_CERTIFICATE, caCertificate);
717+
return this;
718+
}
719+
679720
/**
680721
* @return the {@link ConnectionOptions}
681722
*/
@@ -1300,6 +1341,10 @@ String getClientCertificateKey() {
13001341
return getInitialConnectionPropertyValue(CLIENT_KEY);
13011342
}
13021343

1344+
String getCaCertificate() {
1345+
return getInitialConnectionPropertyValue(CA_CERTIFICATE);
1346+
}
1347+
13031348
/**
13041349
* The (custom) user agent string to use for this connection. If <code>null</code>, then the
13051350
* default JDBC user agent string will be used.

‎java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionProperties.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import static com.google.cloud.spanner.connection.ConnectionOptions.AUTO_BATCH_DML_UPDATE_COUNT_VERIFICATION_PROPERTY_NAME;
2323
import static com.google.cloud.spanner.connection.ConnectionOptions.AUTO_PARTITION_MODE_PROPERTY_NAME;
2424
import static com.google.cloud.spanner.connection.ConnectionOptions.BATCH_DML_UPDATE_COUNT_PROPERTY_NAME;
25+
import static com.google.cloud.spanner.connection.ConnectionOptions.CA_CERTIFICATE_PROPERTY_NAME;
2526
import static com.google.cloud.spanner.connection.ConnectionOptions.CHANNEL_PROVIDER_PROPERTY_NAME;
2627
import static com.google.cloud.spanner.connection.ConnectionOptions.CLIENT_CERTIFICATE_PROPERTY_NAME;
2728
import static com.google.cloud.spanner.connection.ConnectionOptions.CLIENT_KEY_PROPERTY_NAME;
@@ -42,6 +43,7 @@
4243
import static com.google.cloud.spanner.connection.ConnectionOptions.DEFAULT_AUTO_BATCH_DML_UPDATE_COUNT_VERIFICATION;
4344
import static com.google.cloud.spanner.connection.ConnectionOptions.DEFAULT_AUTO_PARTITION_MODE;
4445
import static com.google.cloud.spanner.connection.ConnectionOptions.DEFAULT_BATCH_DML_UPDATE_COUNT;
46+
import static com.google.cloud.spanner.connection.ConnectionOptions.DEFAULT_CA_CERTIFICATE;
4547
import static com.google.cloud.spanner.connection.ConnectionOptions.DEFAULT_CHANNEL_PROVIDER;
4648
import static com.google.cloud.spanner.connection.ConnectionOptions.DEFAULT_CLIENT_CERTIFICATE;
4749
import static com.google.cloud.spanner.connection.ConnectionOptions.DEFAULT_CLIENT_KEY;
@@ -329,6 +331,13 @@ public class ConnectionProperties {
329331
DEFAULT_CLIENT_KEY,
330332
StringValueConverter.INSTANCE,
331333
Context.STARTUP);
334+
static final ConnectionProperty<String> CA_CERTIFICATE =
335+
create(
336+
CA_CERTIFICATE_PROPERTY_NAME,
337+
"Specifies the file path to the server root CA certificate for SSL/TLS validation.",
338+
DEFAULT_CA_CERTIFICATE,
339+
StringValueConverter.INSTANCE,
340+
Context.STARTUP);
332341
static final ConnectionProperty<String> CREDENTIALS_URL =
333342
create(
334343
CREDENTIALS_PROPERTY_NAME,

‎java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/SpannerPool.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,7 @@ static class SpannerPoolKey {
176176
private final boolean enableEndToEndTracing;
177177
private final String clientCertificate;
178178
private final String clientCertificateKey;
179+
private final String caCertificate;
179180
private final SpannerOptions.InstanceType instanceType;
180181
private final Boolean enableDirectAccess;
181182
private final String universeDomain;
@@ -221,6 +222,7 @@ private SpannerPoolKey(ConnectionOptions options) throws IOException {
221222
this.enableEndToEndTracing = options.isEndToEndTracingEnabled();
222223
this.clientCertificate = options.getClientCertificate();
223224
this.clientCertificateKey = options.getClientCertificateKey();
225+
this.caCertificate = options.getCaCertificate();
224226
this.instanceType = options.getInstanceType();
225227
this.enableDirectAccess = options.isEnableDirectAccess();
226228
this.universeDomain = options.getUniverseDomain();
@@ -261,6 +263,7 @@ public boolean equals(Object o) {
261263
&& Objects.equals(this.enableEndToEndTracing, other.enableEndToEndTracing)
262264
&& Objects.equals(this.clientCertificate, other.clientCertificate)
263265
&& Objects.equals(this.clientCertificateKey, other.clientCertificateKey)
266+
&& Objects.equals(this.caCertificate, other.caCertificate)
264267
&& Objects.equals(this.instanceType, other.instanceType)
265268
&& Objects.equals(this.enableDirectAccess, other.enableDirectAccess)
266269
&& Objects.equals(this.universeDomain, other.universeDomain)
@@ -296,6 +299,7 @@ public int hashCode() {
296299
this.enableEndToEndTracing,
297300
this.clientCertificate,
298301
this.clientCertificateKey,
302+
this.caCertificate,
299303
this.instanceType,
300304
this.enableDirectAccess,
301305
this.universeDomain,
@@ -540,6 +544,9 @@ Spanner createSpanner(SpannerPoolKey key, ConnectionOptions options) {
540544
if (key.clientCertificate != null && key.clientCertificateKey != null) {
541545
builder.useClientCert(key.clientCertificate, key.clientCertificateKey);
542546
}
547+
if (key.caCertificate != null) {
548+
builder.setCaCertificate(key.caCertificate);
549+
}
543550
if (key.instanceType != null) {
544551
builder.setType(key.instanceType);
545552
}

0 commit comments

Comments
 (0)