S3 PrivateLink Support.

Controlling access to your VPC endpoints for Amazon S3 by specifying endpointOverride in ClientConfiguration.

Author: wps132230

Docs/ClientConfiguration_Parameters.md

@@ -52,7 +52,14 @@ This value determines the length of time, in milliseconds, to wait before timing
 The retry strategy defaults to exponential backoff. You can override this default by implementing a subclass of RetryStrategy and passing an instance.
 
 ### Endpoint Override
-Do not alter the endpoint.
+This value will override the endpoints the client hits.
+
+Specially, for Amazon S3 and S3 Control clients, you could configure this value to access Amazon S3 interface VPC endpoints (AWS PrivateLink).
+
+For example,
+- use `bucket.vpce-1a2b3c4d-5e6f.s3.us-east-1.vpce.amazonaws.com` to access S3 buckets.
+- use `accesspoint.vpce-1a2b3c4d-5e6f.s3.us-east-1.vpce.amazonaws.com` to access S3 Access Points.
+- use `control.vpce-1a2b3c4d-5e6f.s3.us-east-1.vpce.amazonaws.com` for S3 Control APIs.
 
 ### Proxy Scheme, Host, Port, User Name, and Password
 These settings allow you to configure a proxy for all communication with AWS. Examples of when this functionality might be useful include debugging in conjunction with the Burp suite, or using a proxy to connect to the internet.

aws-cpp-sdk-s3-integration-tests/BucketAndObjectOperationTest.cpp

@@ -8,6 +8,7 @@
 #include <aws/core/client/ClientConfiguration.h>
 #include <aws/core/client/CoreErrors.h>
 #include <aws/core/client/RetryStrategy.h>
+#include <aws/core/client/DefaultRetryStrategy.h>
 #include <aws/core/http/HttpClientFactory.h>
 #include <aws/core/http/HttpClient.h>
 #include <aws/core/utils/crypto/Cipher.h>
@@ -42,6 +43,7 @@
 #include <aws/testing/ProxyConfig.h>
 #include <aws/testing/platform/PlatformTesting.h>
 #include <aws/testing/TestingEnvironment.h>
+#include <aws/testing/mocks/monitoring/TestingMonitoring.h>
 #include <fstream>
 
 #ifdef _WIN32
@@ -76,6 +78,7 @@ namespace
     static std::string BASE_EVENT_STREAM_LARGE_FILE_TEST_BUCKET_NAME = "largeeventstream";
     static std::string BASE_EVENT_STREAM_ERRORS_IN_EVENT_TEST_BUCKET_NAME = "errorsinevent";
     static std::string BASE_CROSS_REGION_BUCKET_NAME = "crossregion";
+    static std::string BASE_ENDPOINT_OVERRIDE_BUCKET_NAME = "endpointoverride";
     static const char* ALLOCATION_TAG = "BucketAndObjectOperationTest";
     static const char* TEST_OBJ_KEY = "TestObjectKey";
     static const char* TEST_NOT_MODIFIED_OBJ_KEY = "TestNotModifiedObjectKey";
@@ -86,6 +89,7 @@ namespace
     //to get around this, this string is url encoded version of "TestUnicode中国Key". At test time, we'll convert it to the unicode string
     static const char* URLENCODED_UNICODE_KEY = "TestUnicode%E4%B8%AD%E5%9B%BDKey";
     static const char* URIESCAPE_KEY = "Esc ape+Me$";
+    static const char* CUSTOM_ENDPOINT_OVERRIDE = "beta.example.com";
 
     static const int TIMEOUT_MAX = 20;
 
@@ -113,6 +117,7 @@ namespace
         AppendUUID(BASE_EVENT_STREAM_LARGE_FILE_TEST_BUCKET_NAME);
         AppendUUID(BASE_EVENT_STREAM_ERRORS_IN_EVENT_TEST_BUCKET_NAME);
         AppendUUID(BASE_CROSS_REGION_BUCKET_NAME);
+        AppendUUID(BASE_ENDPOINT_OVERRIDE_BUCKET_NAME);
     }
 
     class RetryFiveTimesRetryStrategy: public Aws::Client::RetryStrategy
@@ -176,10 +181,13 @@ namespace
             retryClient = Aws::MakeShared<S3Client>(ALLOCATION_TAG,
                     Aws::MakeShared<DefaultAWSCredentialsProviderChain>(ALLOCATION_TAG), config,
                         AWSAuthV4Signer::PayloadSigningPolicy::Never /*signPayloads*/, true /*useVirtualAddressing*/);
+            // Using client side monitoring for endpoint override testing.
+            TestingMonitoringManager::InitTestingMonitoring();
         }
 
         static void TearDownTestCase()
         {
+            TestingMonitoringManager::CleanupTestingMonitoring();
             DeleteBucket(CalculateBucketName(BASE_CREATE_BUCKET_TEST_NAME.c_str()));
             DeleteBucket(CalculateBucketName(BASE_DNS_UNFRIENDLY_TEST_NAME.c_str()));
             DeleteBucket(CalculateBucketName(BASE_LOCATION_BUCKET_TEST_NAME.c_str()));
@@ -1701,6 +1709,115 @@ namespace
         ASSERT_TRUE(deleteBucketOutcome.IsSuccess());
     }
 
+    TEST_F(BucketAndObjectOperationTest, TestCustomEndpointOverride)
+    {
+        // Access Point ARN without dualstack
+        ASSERT_STREQ("myendpoint-123456789012.beta.example.com",
+            S3Endpoint::ForAccessPointArn(S3ARN("arn:aws:s3:us-west-2:123456789012:accesspoint:myendpoint"), "", false /* useDualStack */, "beta.example.com").c_str());
+        // Outpost Access Point ARN without dualstack
+        ASSERT_STREQ("myaccesspoint-123456789012.op-01234567890123456.beta.example.com",
+            S3Endpoint::ForOutpostsArn(S3ARN("arn:aws:s3-outposts:us-west-2:123456789012:outpost:op-01234567890123456:accesspoint:myaccesspoint"), "",
+                false /* useDualStack */, "beta.example.com").c_str());
+
+        Aws::String fullBucketName = CalculateBucketName(BASE_ENDPOINT_OVERRIDE_BUCKET_NAME.c_str());
+        Aws::StringStream ss;
+
+        // Traditional bucket name with virtual addressing
+        ClientConfiguration config;
+        config.region = Aws::Region::US_WEST_2;
+        config.endpointOverride = CUSTOM_ENDPOINT_OVERRIDE;
+        config.retryStrategy = Aws::MakeShared<Aws::Client::DefaultRetryStrategy>(ALLOCATION_TAG, 0 /* don't retry */, 25);
+        S3Client s3ClientWithVirtualAddressing(config, AWSAuthV4Signer::PayloadSigningPolicy::Never, true /*useVirtualAddressing*/);
+
+        ListObjectsRequest listObjectsRequest;
+        listObjectsRequest.SetBucket(fullBucketName);
+        auto listObjectsOutcome = s3ClientWithVirtualAddressing.ListObjects(listObjectsRequest);
+        ASSERT_FALSE(listObjectsOutcome.IsSuccess());
+        ASSERT_EQ(HttpResponseCode::REQUEST_NOT_MADE, listObjectsOutcome.GetError().GetResponseCode());
+        ss << "https://" << fullBucketName << "." << CUSTOM_ENDPOINT_OVERRIDE;
+        ASSERT_STREQ(ss.str().c_str(), TestingMonitoringMetrics::s_lastUriString.c_str());
+        ASSERT_STREQ("s3", TestingMonitoringMetrics::s_lastSigningServiceName.c_str());
+
+        // Access Point Arn with virtual addressing
+        listObjectsRequest.SetBucket("arn:aws:s3:us-west-2:123456789012:accesspoint:myendpoint");
+        listObjectsOutcome = s3ClientWithVirtualAddressing.ListObjects(listObjectsRequest);
+        ASSERT_FALSE(listObjectsOutcome.IsSuccess());
+        ASSERT_EQ(HttpResponseCode::REQUEST_NOT_MADE, listObjectsOutcome.GetError().GetResponseCode());
+        ss.str("");
+        ss << "https://myendpoint-123456789012." << CUSTOM_ENDPOINT_OVERRIDE;
+        ASSERT_STREQ(ss.str().c_str(), TestingMonitoringMetrics::s_lastUriString.c_str());
+        ASSERT_STREQ("s3", TestingMonitoringMetrics::s_lastSigningServiceName.c_str());
+
+        // Outposts Access Point Arn with virtual addressing
+        listObjectsRequest.SetBucket("arn:aws:s3-outposts:us-west-2:123456789012:outpost:op-01234567890123456:accesspoint:myaccesspoint");
+        listObjectsOutcome = s3ClientWithVirtualAddressing.ListObjects(listObjectsRequest);
+        ASSERT_FALSE(listObjectsOutcome.IsSuccess());
+        ASSERT_EQ(HttpResponseCode::REQUEST_NOT_MADE, listObjectsOutcome.GetError().GetResponseCode());
+        ss.str("");
+        ss << "https://myaccesspoint-123456789012.op-01234567890123456." << CUSTOM_ENDPOINT_OVERRIDE;
+        ASSERT_STREQ(ss.str().c_str(), TestingMonitoringMetrics::s_lastUriString.c_str());
+        ASSERT_STREQ("s3-outposts", TestingMonitoringMetrics::s_lastSigningServiceName.c_str());
+
+        // ListBuckets
+        auto listBucketsOutcome = s3ClientWithVirtualAddressing.ListBuckets();
+        ASSERT_FALSE(listBucketsOutcome.IsSuccess());
+        ASSERT_EQ(HttpResponseCode::REQUEST_NOT_MADE, listBucketsOutcome.GetError().GetResponseCode());
+        ss.str("");
+        ss << "https://" << CUSTOM_ENDPOINT_OVERRIDE;
+        ASSERT_STREQ(ss.str().c_str(), TestingMonitoringMetrics::s_lastUriString.c_str());
+        ASSERT_STREQ("s3", TestingMonitoringMetrics::s_lastSigningServiceName.c_str());
+
+        // Tradition bucket name with path addressing
+        S3Client s3ClientWithPathAddressing(config, AWSAuthV4Signer::PayloadSigningPolicy::Never, false /*useVirtualAddressing*/);
+        listObjectsRequest.SetBucket(fullBucketName);
+        listObjectsOutcome = s3ClientWithPathAddressing.ListObjects(listObjectsRequest);
+        ASSERT_FALSE(listObjectsOutcome.IsSuccess());
+        ASSERT_EQ(HttpResponseCode::REQUEST_NOT_MADE, listObjectsOutcome.GetError().GetResponseCode());
+        ss.str("");
+        ss << "https://" << CUSTOM_ENDPOINT_OVERRIDE << "/" << fullBucketName;
+        ASSERT_STREQ(ss.str().c_str(), TestingMonitoringMetrics::s_lastUriString.c_str());
+        ASSERT_STREQ("s3", TestingMonitoringMetrics::s_lastSigningServiceName.c_str());
+
+        // Use arn region, Access Point Arn with virtual addressing
+        Aws::String awsS3UseArnRegion = Aws::Environment::GetEnv("AWS_S3_USE_ARN_REGION");
+        Aws::Environment::SetEnv("AWS_S3_USE_ARN_REGION", "true", 1);
+        config.region = Aws::Region::EU_WEST_1;
+        S3Client s3ClientInEuWest1(config, AWSAuthV4Signer::PayloadSigningPolicy::Never, true /*useVirtualAddressing*/);
+        if (awsS3UseArnRegion.empty())
+        {
+            Aws::Environment::UnSetEnv("AWS_S3_USE_ARN_REGION");
+        }
+        else
+        {
+            Aws::Environment::SetEnv("AWS_S3_USE_ARN_REGION", awsS3UseArnRegion.c_str(), 1);
+        }
+        listObjectsRequest.SetBucket("arn:aws:s3:us-west-2:123456789012:accesspoint:myendpoint");
+        listObjectsOutcome = s3ClientInEuWest1.ListObjects(listObjectsRequest);
+        ASSERT_FALSE(listObjectsOutcome.IsSuccess());
+        ASSERT_EQ(HttpResponseCode::REQUEST_NOT_MADE, listObjectsOutcome.GetError().GetResponseCode());
+        ss.str("");
+        ss << "https://myendpoint-123456789012." << CUSTOM_ENDPOINT_OVERRIDE;
+        ASSERT_STREQ(ss.str().c_str(), TestingMonitoringMetrics::s_lastUriString.c_str());
+        ASSERT_STREQ("s3", TestingMonitoringMetrics::s_lastSigningServiceName.c_str());
+        ASSERT_STREQ("us-west-2", TestingMonitoringMetrics::s_lastSigningRegion.c_str());
+
+        // Failure case, dualstack endpoint is not compatible with custom endpoint override.
+        config.region = Aws::Region::US_WEST_2;
+        config.useDualStack = true;
+        S3Client s3ClientWithDualStack(config, AWSAuthV4Signer::PayloadSigningPolicy::Never, true /*useVirtualAddressing*/);
+        listObjectsRequest.SetBucket(fullBucketName);
+        listObjectsOutcome = s3ClientWithDualStack.ListObjects(listObjectsRequest);
+        ASSERT_FALSE(listObjectsOutcome.IsSuccess());
+        ASSERT_EQ(HttpResponseCode::REQUEST_NOT_MADE, listObjectsOutcome.GetError().GetResponseCode());
+        ASSERT_EQ(S3Errors::VALIDATION, listObjectsOutcome.GetError().GetErrorType());
+
+        listObjectsRequest.SetBucket("arn:aws:s3-outposts:us-west-2:123456789012:outpost:op-01234567890123456:accesspoint:myaccesspoint");
+        listObjectsOutcome = s3ClientWithDualStack.ListObjects(listObjectsRequest);
+        ASSERT_FALSE(listObjectsOutcome.IsSuccess());
+        ASSERT_EQ(HttpResponseCode::REQUEST_NOT_MADE, listObjectsOutcome.GetError().GetResponseCode());
+        ASSERT_EQ(S3Errors::VALIDATION, listObjectsOutcome.GetError().GetErrorType());
+    }
+
     TEST_F(BucketAndObjectOperationTest, TestS3AccessPointARNValidation)
     {
         // The followings are examples for valid S3 ARN:
@@ -1837,5 +1954,4 @@ namespace
         ASSERT_STREQ("access-point-name-123456789120.outpost-id.s3-outposts.cn-north-1.amazonaws.com.cn",
             S3Endpoint::ForOutpostsArn(S3ARN("arn:aws-cn:s3-outposts:cn-north-1:123456789120:outpost:outpost-id:accesspoint:access-point-name"), "").c_str());
     }
-
 }

aws-cpp-sdk-s3/include/aws/s3/S3Endpoint.h

@@ -29,15 +29,18 @@ namespace S3Endpoint
    * @param arn The S3 Access Point ARN
    * @param regionNameOverride Override region name in ARN if it's not empty
    * @param useDualStack Using dual-stack endpoint if true
+   * @param endpointOverride Override endpoint if it's not empty
    */
-  AWS_S3_API Aws::String ForAccessPointArn(const S3ARN& arn, const Aws::String& regionNameOverride = "", bool useDualStack = false);
+  AWS_S3_API Aws::String ForAccessPointArn(const S3ARN& arn, const Aws::String& regionNameOverride = "", bool useDualStack = false, const Aws::String& endpointOverride = "");
 
   /**
    * Compute endpoint based on Outposts ARN.
    * @param arn The S3 Outposts ARN
    * @param regionNameOverride Override region name in ARN if it's not empty
+   * @param useDualStack Using dual-stack endpoint if true
+   * @param endpointOverride Override endpoint if it's not empty
    */
-  AWS_S3_API Aws::String ForOutpostsArn(const S3ARN& arn, const Aws::String& regionNameOverride = "");
+  AWS_S3_API Aws::String ForOutpostsArn(const S3ARN& arn, const Aws::String& regionNameOverride = "", bool useDualStack = false, const Aws::String& endpointOverride = "");
 } // namespace S3Endpoint
 } // namespace S3
 } // namespace Aws

aws-cpp-sdk-s3/source/S3Client.cpp

@@ -4013,6 +4013,12 @@ Aws::String S3Client::GeneratePresignedUrlWithSSEC(const Aws::String& bucket, co
 
 ComputeEndpointOutcome S3Client::ComputeEndpointString(const Aws::String& bucketOrArn) const
 {
+    if (m_useDualStack && m_useCustomEndpoint)
+    {
+        return ComputeEndpointOutcome(Aws::Client::AWSError<S3Errors>(S3Errors::VALIDATION, "VALIDATION",
+            "Dual-stack endpoint is incompatible with a custom endpoint override.", false));
+    }
+
     Aws::StringStream ss;
     ss << m_scheme << "://";
     Aws::String bucket = bucketOrArn;
@@ -4021,12 +4027,6 @@ ComputeEndpointOutcome S3Client::ComputeEndpointString(const Aws::String& bucket
 
     if (arn)
     {
-        if (m_useCustomEndpoint)
-        {
-            return ComputeEndpointOutcome(Aws::Client::AWSError<S3Errors>(S3Errors::VALIDATION, "VALIDATION",
-                "Custom endpoint is not compatible with Access Point ARN or Outposts ARN in Bucket field.", false));
-        }
-
         if (!m_useVirtualAddressing)
         {
             return ComputeEndpointOutcome(Aws::Client::AWSError<S3Errors>(S3Errors::VALIDATION, "VALIDATION",
@@ -4041,7 +4041,7 @@ ComputeEndpointOutcome S3Client::ComputeEndpointString(const Aws::String& bucket
         signerRegion = m_useArnRegion ? arn.GetRegion() : signerRegion;
         if (arn.GetResourceType() == ARNResourceType::ACCESSPOINT)
         {
-            ss << S3Endpoint::ForAccessPointArn(arn, m_useArnRegion ? "" : m_region, m_useDualStack);
+            ss << S3Endpoint::ForAccessPointArn(arn, m_useArnRegion ? "" : m_region, m_useDualStack, m_useCustomEndpoint ? m_baseUri : "");
             return ComputeEndpointOutcome(ComputeEndpointResult(ss.str(), signerRegion, SERVICE_NAME));
         }
         else if (arn.GetResourceType() == ARNResourceType::OUTPOST)
@@ -4051,7 +4051,7 @@ ComputeEndpointOutcome S3Client::ComputeEndpointString(const Aws::String& bucket
                 return ComputeEndpointOutcome(Aws::Client::AWSError<S3Errors>(S3Errors::VALIDATION, "VALIDATION",
                     "Outposts Access Points do not support dualstack right now.", false));
             }
-            ss << S3Endpoint::ForOutpostsArn(arn, m_useArnRegion ? "" : m_region);
+            ss << S3Endpoint::ForOutpostsArn(arn, m_useArnRegion ? "" : m_region, m_useDualStack, m_useCustomEndpoint ? m_baseUri : "");
             return ComputeEndpointOutcome(ComputeEndpointResult(ss.str(), signerRegion, "s3-outposts"));
         }
     }
@@ -4074,6 +4074,11 @@ ComputeEndpointOutcome S3Client::ComputeEndpointString(const Aws::String& bucket
 
 ComputeEndpointOutcome S3Client::ComputeEndpointString() const
 {
+    if (m_useDualStack && m_useCustomEndpoint)
+    {
+        return ComputeEndpointOutcome(Aws::Client::AWSError<S3Errors>(S3Errors::VALIDATION, "VALIDATION",
+            "Dual-stack endpoint is incompatible with a custom endpoint override.", false));
+    }
     Aws::StringStream ss;
     ss << m_scheme << "://" << m_baseUri;
     return ComputeEndpointOutcome(ComputeEndpointResult(ss.str(), Aws::Region::ComputeSignerRegion(m_region), SERVICE_NAME));

aws-cpp-sdk-s3/source/S3Endpoint.cpp

@@ -27,12 +27,19 @@ namespace S3Endpoint
   static const int US_EAST_1_HASH = Aws::Utils::HashingUtils::HashString("us-east-1");
   static const int AWS_GLOBAL_HASH = Aws::Utils::HashingUtils::HashString("aws-global");
 
-  Aws::String ForAccessPointArn(const S3ARN& arn, const Aws::String& regionNameOverride, bool useDualStack)
+  Aws::String ForAccessPointArn(const S3ARN& arn, const Aws::String& regionNameOverride, bool useDualStack, const Aws::String& endpointOverride)
   {
+    Aws::StringStream ss;
+
+    if (!endpointOverride.empty())
+    {
+      ss << arn.GetResourceId() << "-" << arn.GetAccountId() << "." << endpointOverride;
+      return ss.str();
+    }
+
     const Aws::String& region = regionNameOverride.empty() ? arn.GetRegion() : regionNameOverride;
     auto hash = Aws::Utils::HashingUtils::HashString(region.c_str());
 
-    Aws::StringStream ss;
     ss << arn.GetResourceId() << "-" << arn.GetAccountId() << ".s3-accesspoint.";
     if (useDualStack)
     {
@@ -48,12 +55,21 @@ namespace S3Endpoint
     return ss.str();
   }
 
-  Aws::String ForOutpostsArn(const S3ARN& arn, const Aws::String& regionNameOverride)
+  Aws::String ForOutpostsArn(const S3ARN& arn, const Aws::String& regionNameOverride, bool useDualStack, const Aws::String& endpointOverride)
   {
+    AWS_UNREFERENCED_PARAM(useDualStack);
+    assert(!useDualStack);
+    Aws::StringStream ss;
+
+    if (!endpointOverride.empty())
+    {
+      ss << arn.GetSubResourceId() << "-" << arn.GetAccountId() << "." << arn.GetResourceId() << "." << endpointOverride;
+      return ss.str();
+    }
+
     const Aws::String& region = regionNameOverride.empty() ? arn.GetRegion() : regionNameOverride;
     auto hash = Aws::Utils::HashingUtils::HashString(region.c_str());
 
-    Aws::StringStream ss;
     ss << arn.GetSubResourceId() << "-" << arn.GetAccountId() << "." << arn.GetResourceId() << ".s3-outposts." << region << "." << "amazonaws.com";
 
     if (hash == CN_NORTH_1_HASH || hash == CN_NORTHWEST_1_HASH)