Transfer MPU Validations, Part Size Updates, and passing all request fields to low level request (#4096)

* Increase multipart upload default part size to 8MB (#4032)

* Update Generator to add s3 project to give unit test access

* Add missing fields to Transfer Utility request objects (#4056)

* Remove response related tests

* Fix issue with HeadersCollection in ResponseMapperTests

* Add additional validation to UploadPartRequests in Transfer Utility (#4083)

* create tuabortmultipartuploadrequest (#4093)

---------

Co-authored-by: Philippe El Asmar <[email protected]>
Co-authored-by: Phil Asmar <[email protected]>

Author: GarrettBeatty

generator/.DevConfigs/252dad9f-d2a9-4d49-bff8-000924f0adc3.json

@@ -0,0 +1,11 @@
+{
+  "services": [
+    {
+      "serviceName": "S3",
+      "type": "minor",
+      "changeLogMessages": [
+        "Increasing the default part size for S3 multipart upload from 5MB to 8MB when no part size is specified. This will reduce the number of API calls for multipart uploads."
+      ]
+    }
+  ]
+}

generator/.DevConfigs/252dad9f-d2a9-4d49-bff8-000924f0adt0.json

@@ -0,0 +1,11 @@
+{
+  "services": [
+    {
+      "serviceName": "S3",
+      "type": "patch",
+      "changeLogMessages": [
+        "Update AssemblyInfo to give S3 Unit Test project access to internals."
+      ]
+    }
+  ]
+}

generator/.DevConfigs/49ef8a70-bb30-4cc4-a8b5-92de4f6068c1.json

@@ -0,0 +1,12 @@
+{
+  "services": [
+    {
+      "serviceName": "S3",
+      "type": "patch",
+      "changeLogMessages": [
+        "Fixed issue where PartSize and IsLastPart fields were not properly set on Transfer Utility Upload Part Request.",
+        "Add additional validations for Transfer Utility requests to ensure Upload Parts have the proper Content Length and File Offsets."
+      ]
+    }
+  ]
+}

generator/.DevConfigs/9d07dc1e-d82d-4f94-8700-c7b57f872041.json

@@ -0,0 +1,11 @@
+{
+  "services": [
+    {
+      "serviceName": "S3",
+      "type": "patch",
+      "changeLogMessages": [
+        "Create AbortMultipartUploads api that takes in TransferUtilityAbortMultipartUploadRequest."
+      ]
+    }
+  ]
+}

generator/.DevConfigs/9d07dc1e-d82d-4f94-8700-c7b57f87205d.json

@@ -0,0 +1,11 @@
+{
+  "services": [
+    {
+      "serviceName": "S3",
+      "type": "patch",
+      "changeLogMessages": [
+        "Add missing fields to Transfer Utility request objects. ContentType on TransferUtilityUploadRequest and TransferUtilityUploadDirectoryRequest now directly updates the ContentType header, instead of being a separate field on those objects."
+      ]
+    }
+  ]
+}

generator/ServiceClientGeneratorLib/Generators/SourceFiles/AssemblyInfo.cs

@@ -15,7 +15,7 @@ namespace ServiceClientGenerator.Generators.SourceFiles
     /// Class to produce the template output
     /// </summary>
 
-    #line 1 "C:\codebase\v4\aws-sdk-net-v4\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
+    #line 1 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
     [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.TextTemplating", "17.0.0.0")]
     public partial class AssemblyInfo : BaseGenerator
     {
@@ -36,43 +36,43 @@ public override string TransformText()
 // associated with an assembly.
 [assembly: AssemblyTitle(""");
 
-            #line 12 "C:\codebase\v4\aws-sdk-net-v4\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
+            #line 12 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
             this.Write(this.ToStringHelper.ToStringWithCulture(this.Config.AssemblyTitle));
 
             #line default
             #line hidden
             this.Write("\")]\r\n#if BCL\r\n[assembly: AssemblyDescription(\"");
 
-            #line 14 "C:\codebase\v4\aws-sdk-net-v4\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
+            #line 14 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
             this.Write(this.ToStringHelper.ToStringWithCulture(this.Config.AssemblyDescription(versionIdentifier: "4.7.2")));
 
             #line default
             #line hidden
             this.Write("\")]\r\n#elif NETSTANDARD20\r\n[assembly: AssemblyDescription(\"");
 
-            #line 16 "C:\codebase\v4\aws-sdk-net-v4\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
+            #line 16 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
             this.Write(this.ToStringHelper.ToStringWithCulture(this.Config.AssemblyDescription(versionIdentifier: "NetStandard 2.0")));
 
             #line default
             #line hidden
             this.Write("\")]\r\n#elif NETCOREAPP3_1\r\n[assembly: AssemblyDescription(\"");
 
-            #line 18 "C:\codebase\v4\aws-sdk-net-v4\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
+            #line 18 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
             this.Write(this.ToStringHelper.ToStringWithCulture(this.Config.AssemblyDescription(versionIdentifier: ".NET Core 3.1")));
 
             #line default
             #line hidden
             this.Write("\")]\r\n#elif NET8_0\r\n[assembly: AssemblyDescription(\"");
 
-            #line 20 "C:\codebase\v4\aws-sdk-net-v4\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
+            #line 20 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
             this.Write(this.ToStringHelper.ToStringWithCulture(this.Config.AssemblyDescription(versionIdentifier: ".NET 8.0")));
 
             #line default
             #line hidden
             this.Write("\")]\r\n#else\r\n#error Unknown platform constant - unable to set correct AssemblyDesc" +
                     "ription\r\n#endif\r\n\r\n");
 
-            #line 25 "C:\codebase\v4\aws-sdk-net-v4\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
+            #line 25 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
  if (this.Config.AssemblyTitle=="AWSSDK.DynamoDBv2") { 
 
             #line default
@@ -81,7 +81,22 @@ public override string TransformText()
 [assembly: InternalsVisibleTo(""AWSSDK.UnitTests.NetFramework, PublicKey=0024000004800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4"")]
 ");
 
-            #line 28 "C:\codebase\v4\aws-sdk-net-v4\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
+            #line 28 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
+ } 
+            
+            #line default
+            #line hidden
+            
+            #line 29 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
+ if (this.Config.AssemblyTitle=="AWSSDK.S3") { 
+            
+            #line default
+            #line hidden
+            this.Write(@"[assembly: InternalsVisibleTo(""AWSSDK.UnitTests.S3.NetFramework, PublicKey=0024000004800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4"")]
+[assembly: InternalsVisibleTo(""AWSSDK.UnitTests.NetFramework, PublicKey=0024000004800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4"")]
+");
+            
+            #line 32 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
  } 
 
             #line default
@@ -110,14 +125,14 @@ public override string TransformText()
 // [assembly: AssemblyVersion(""1.0.*"")]
 [assembly: AssemblyVersion(""");
 
-            #line 51 "C:\codebase\v4\aws-sdk-net-v4\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
+            #line 55 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
             this.Write(this.ToStringHelper.ToStringWithCulture(this.Config.ServiceVersion));
 
             #line default
             #line hidden
             this.Write("\")]\r\n[assembly: AssemblyFileVersion(\"");
 
-            #line 52 "C:\codebase\v4\aws-sdk-net-v4\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
+            #line 56 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
             this.Write(this.ToStringHelper.ToStringWithCulture(this.Config.ServiceFileVersion));
 
             #line default

generator/ServiceClientGeneratorLib/Generators/SourceFiles/AssemblyInfo.tt

@@ -26,6 +26,10 @@ using System.Runtime.CompilerServices;
 [assembly: InternalsVisibleTo("AWSSDK.UnitTests.DynamoDBv2.NetFramework, PublicKey=0024000004800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4")]
 [assembly: InternalsVisibleTo("AWSSDK.UnitTests.NetFramework, PublicKey=0024000004800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4")]
 <# } #>
+<# if (this.Config.AssemblyTitle=="AWSSDK.S3") { #>
+[assembly: InternalsVisibleTo("AWSSDK.UnitTests.S3.NetFramework, PublicKey=0024000004800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4")]
+[assembly: InternalsVisibleTo("AWSSDK.UnitTests.NetFramework, PublicKey=0024000004800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4")]
+<# } #>
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyProduct("Amazon Web Services SDK for .NET")]
 [assembly: AssemblyCompany("Amazon.com, Inc")]

generator/ServiceModels/_manifest.json

@@ -60,7 +60,8 @@
                 "Custom\\Runtime\\TestResponses\\*.txt",
                 "Custom\\Runtime\\EventStreams\\test_vectors\\*",
                 "Custom\\Runtime\\TestEndpoints\\*.json",
-                "Custom\\TestTools\\ComparerTest.json"
+                "Custom\\TestTools\\ComparerTest.json",
+                "..\\Services\\S3\\UnitTests\\Custom\\EmbeddedResource\\*"
             ],
             "packageReferences": [
                 {

sdk/src/Services/S3/Custom/Model/HeadersCollection.cs

@@ -152,6 +152,21 @@ internal bool IsSetContentType()
             return !string.IsNullOrEmpty(this.ContentType);
         }
 
+        /// <summary>
+        /// The language that the content is in. For more information,
+        /// see <a href="https://www.rfc-editor.org/rfc/rfc9110.html#name-content-language">https://www.rfc-editor.org/rfc/rfc9110.html#name-content-language</a>.
+        /// </summary>
+        public string ContentLanguage
+        {
+            get { return this["Content-Language"]; }
+            set { this["Content-Language"] = value; }
+        }
+
+        internal bool IsSetContentLanguage()
+        {
+            return !string.IsNullOrEmpty(this.ContentLanguage);
+        }
+
         /// <summary>
         /// <para>
         /// The date and time at which the object is no longer cacheable. For more information,

sdk/src/Services/S3/Custom/Transfer/BaseDownloadRequest.cs

@@ -24,6 +24,7 @@
 using System.Text;
 
 using Amazon.Runtime.Internal;
+using Amazon.S3.Model;
 
 namespace Amazon.S3.Transfer
 {
@@ -45,6 +46,11 @@ public abstract class BaseDownloadRequest
 
         private RequestPayer requestPayer;
 
+        private string expectedBucketOwner;
+        private string ifMatch;
+        private string ifNoneMatch;
+        private ResponseHeaderOverrides responseHeaders;
+        
         /// <summary>
         /// 	Gets or sets the name of the bucket.
         /// </summary>
@@ -66,7 +72,7 @@ public string BucketName
         /// </returns>
         internal bool IsSetBucketName()
         {
-            return !System.String.IsNullOrEmpty(this.bucketName);
+            return !String.IsNullOrEmpty(this.bucketName);
         }
 
 
@@ -91,7 +97,7 @@ public string Key
         /// </returns>
         internal bool IsSetKey()
         {
-            return !System.String.IsNullOrEmpty(this.key);
+            return !String.IsNullOrEmpty(this.key);
         }
 
         /// <summary>
@@ -112,7 +118,7 @@ public string VersionId
         /// <returns>true if VersionId property is set.</returns>
         internal bool IsSetVersionId()
         {
-            return !System.String.IsNullOrEmpty(this.versionId);
+            return !String.IsNullOrEmpty(this.versionId);
         }        
 
         /// <summary>
@@ -220,5 +226,109 @@ public RequestPayer RequestPayer
             get { return this.requestPayer; }
             set { this.requestPayer = value; }
         }
+        
+        /// <summary>
+        /// Gets and sets the property ExpectedBucketOwner. 
+        /// <para>
+        /// The account ID of the expected bucket owner. If the account ID that you provide does
+        /// not match the actual owner of the bucket, the request fails with the HTTP status code
+        /// <c>403 Forbidden</c> (access denied).
+        /// </para>
+        /// </summary>
+        public string ExpectedBucketOwner
+        {
+            get { return this.expectedBucketOwner; }
+            set { this.expectedBucketOwner = value; }
+        }
+
+        /// <summary>
+        /// Checks to see if ExpectedBucketOwner is set.
+        /// </summary>
+        /// <returns>true, if ExpectedBucketOwner property is set.</returns>
+        internal bool IsSetExpectedBucketOwner()
+        {
+            return !String.IsNullOrEmpty(this.expectedBucketOwner);
+        }
+        
+        /// <summary>
+        /// Gets and sets the property IfMatch. 
+        /// <para>
+        /// Return the object only if its entity tag (ETag) is the same as the one specified in this header;
+        /// otherwise, return a <c>412 Precondition Failed</c> error.
+        /// </para>
+        /// <para>
+        /// If both of the <c>If-Match</c> and <c>If-Unmodified-Since</c> headers are present in the request as follows:
+        /// <c>If-Match</c> condition evaluates to <c>true</c>, and; <c>If-Unmodified-Since</c> condition evaluates to <c>false</c>;
+        /// then, S3 returns <c>200 OK</c> and the data requested.
+        /// </para>
+        /// <para>
+        /// For more information about conditional requests, see <see href="https://tools.ietf.org/html/rfc7232">RFC 7232</see>.
+        /// </para>
+        /// The <see cref="IfMatch"/> property is equivalent to the <see cref="GetObjectRequest.EtagToMatch"/>.
+        /// </summary>
+        public string IfMatch
+        {
+            get { return this.ifMatch; }
+            set { this.ifMatch = value; }
+        }
+
+        /// <summary>
+        /// Checks to see if IfMatch is set.
+        /// </summary>
+        /// <returns>true, if IfMatch property is set.</returns>
+        internal bool IsSetIfMatch()
+        {
+            return !String.IsNullOrEmpty(this.ifMatch);
+        }
+
+        /// <summary>
+        /// Gets and sets the property IfNoneMatch. 
+        /// <para>
+        /// Return the object only if its entity tag (ETag) is different from the one specified in this header;
+        /// otherwise, return a <c>304 Not Modified</c> error.
+        /// </para>
+        /// <para>
+        /// If both of the <c>If-None-Match</c> and <c>If-Modified-Since</c> headers are present in the request as follows:
+        /// <c> If-None-Match</c> condition evaluates to <c>false</c>, and; <c>If-Modified-Since</c> condition evaluates to <c>true</c>;
+        /// then, S3 returns <c>304 Not Modified</c> HTTP status code.
+        /// </para>
+        /// <para>
+        /// For more information about conditional requests, see <see href="https://tools.ietf.org/html/rfc7232">RFC 7232</see>.
+        /// </para>
+        /// The <see cref="IfNoneMatch"/> property is equivalent to the <see cref="GetObjectRequest.EtagToNotMatch"/>.
+        /// </summary>
+        public string IfNoneMatch
+        {
+            get { return this.ifNoneMatch; }
+            set { this.ifNoneMatch = value; }
+        }
+
+        /// <summary>
+        /// Checks to see if IfNoneMatch is set.
+        /// </summary>
+        /// <returns>true, if IfNoneMatch property is set.</returns>
+        internal bool IsSetIfNoneMatch()
+        {
+            return !String.IsNullOrEmpty(this.ifNoneMatch);
+        }
+
+        /// <summary>
+        /// A set of response headers that should be returned with the object.
+        /// </summary>
+        public ResponseHeaderOverrides ResponseHeaderOverrides
+        {
+            get
+            {
+                if (this.responseHeaders == null)
+                {
+                    this.responseHeaders = new ResponseHeaderOverrides();
+                }
+                return this.responseHeaders;
+            }
+            set
+            {
+                this.responseHeaders = value;
+            }
+        }
     }
 }