Expose CRT file IO options

Author: hssyoo

awscli/customizations/s3/factory.py

@@ -138,6 +138,14 @@ def _create_crt_client(self, params, runtime_config):
             create_crt_client_kwargs['crt_credentials_provider'] = (
                 crt_credentials_provider
             )
+        fio_options = {}
+        if (val := runtime_config.get('should_stream')) is not None:
+            fio_options['should_stream'] = val
+        if (val := runtime_config.get('disk_throughput')) is not None:
+            fio_options['disk_throughput_gbps'] = val
+        if (val := runtime_config.get('direct_io')) is not None:
+            fio_options['direct_io'] = val
+        create_crt_client_kwargs['fio_options'] = fio_options
 
         return create_s3_crt_client(**create_crt_client_kwargs)
 

awscli/customizations/s3/transferconfig.py

@@ -15,6 +15,7 @@
 # commands.
 import logging
 
+from botocore.utils import ensure_boolean
 from s3transfer.manager import TransferConfig
 
 from awscli.customizations.s3 import constants
@@ -31,6 +32,9 @@
     'preferred_transfer_client': constants.AUTO_RESOLVE_TRANSFER_CLIENT,
     'target_bandwidth': None,
     'io_chunksize': 256 * 1024,
+    'should_stream': None,
+    'disk_throughput': None,
+    'direct_io': None,
 }
 
 
@@ -47,9 +51,18 @@ class RuntimeConfig:
         'max_bandwidth',
         'target_bandwidth',
         'io_chunksize',
+        'disk_throughput',
+    ]
+    HUMAN_READABLE_SIZES = [
+        'multipart_chunksize',
+        'multipart_threshold',
+        'io_chunksize',
+    ]
+    HUMAN_READABLE_RATES = [
+        'max_bandwidth',
+        'target_bandwidth',
+        'disk_throughput',
     ]
-    HUMAN_READABLE_SIZES = ['multipart_chunksize', 'multipart_threshold', 'io_chunksize']
-    HUMAN_READABLE_RATES = ['max_bandwidth', 'target_bandwidth']
     SUPPORTED_CHOICES = {
         'preferred_transfer_client': [
             constants.AUTO_RESOLVE_TRANSFER_CLIENT,
@@ -62,6 +75,7 @@ class RuntimeConfig:
             'default': constants.CLASSIC_TRANSFER_CLIENT
         }
     }
+    BOOLEANS = ['should_stream', 'direct_io']
 
     @staticmethod
     def defaults():
@@ -83,6 +97,7 @@ def build_config(self, **kwargs):
             runtime_config.update(kwargs)
         self._convert_human_readable_sizes(runtime_config)
         self._convert_human_readable_rates(runtime_config)
+        self._convert_booleans(runtime_config)
         self._resolve_choice_aliases(runtime_config)
         self._validate_config(runtime_config)
         return runtime_config
@@ -116,6 +131,12 @@ def _convert_human_readable_rates(self, runtime_config):
                         'second (e.g. 10Mb/s or 800Kb/s)' % value
                     )
 
+    def _convert_booleans(self, runtime_config):
+        for attr in self.BOOLEANS:
+            value = runtime_config.get(attr)
+            if value is not None:
+                runtime_config[attr] = ensure_boolean(value)
+
     def _human_readable_rate_to_int(self, value):
         # The human_readable_to_int() utility only supports integers (e.g. 1024)
         # as strings and human readable sizes (e.g. 10MB, 5GB). It does not

awscli/s3transfer/crt.py

@@ -33,6 +33,7 @@
 )
 from awscrt.s3 import (
     S3Client,
+    S3FileIoOptions,
     S3RequestTlsMode,
     S3RequestType,
     S3ResponseError,
@@ -44,7 +45,7 @@
 from botocore.exceptions import NoCredentialsError
 from botocore.useragent import register_feature_id
 from botocore.utils import ArnParser, InvalidArnException, is_s3express_bucket
-from s3transfer.constants import FULL_OBJECT_CHECKSUM_ARGS, MB
+from s3transfer.constants import FULL_OBJECT_CHECKSUM_ARGS, GB, MB
 from s3transfer.exceptions import TransferNotDoneError
 from s3transfer.futures import BaseTransferFuture, BaseTransferMeta
 from s3transfer.utils import CallArgs, OSUtils, get_callbacks
@@ -87,6 +88,7 @@ def create_s3_crt_client(
     part_size=8 * MB,
     use_ssl=True,
     verify=None,
+    fio_options=None,
 ):
     """
     :type region: str
@@ -153,6 +155,9 @@ def create_s3_crt_client(
     target_gbps = _get_crt_throughput_target_gbps(
         provided_throughput_target_bytes=target_throughput
     )
+    fio_options = fio_options or {}
+    if disk_throughput := fio_options.get('disk_throughput_gbps'):
+        fio_options['disk_throughput_gbps'] = disk_throughput * 8 / GB
     return S3Client(
         bootstrap=bootstrap,
         region=region,
@@ -162,6 +167,7 @@ def create_s3_crt_client(
         tls_connection_options=tls_connection_options,
         throughput_target_gbps=target_gbps,
         enable_s3express=True,
+        fio_options=S3FileIoOptions(**fio_options),
     )