diff --git a/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsPythonDependency.java b/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsPythonDependency.java index e1a5a6b0d..90aa2e0fe 100644 --- a/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsPythonDependency.java +++ b/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsPythonDependency.java @@ -22,7 +22,7 @@ private AwsPythonDependency() {} */ public static final PythonDependency SMITHY_AWS_CORE = new PythonDependency( "smithy_aws_core", - "~=0.1.0", + "~=0.2.0", PythonDependency.Type.DEPENDENCY, false); } diff --git a/codegen/build.gradle.kts b/codegen/build.gradle.kts index adbdc31a0..c8f736a5d 100644 --- a/codegen/build.gradle.kts +++ b/codegen/build.gradle.kts @@ -15,5 +15,5 @@ allprojects { group = "software.amazon.smithy.python" - version = "0.0.1" + version = "0.1.0" } diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java index 026e4dfe2..95c6e0d5a 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java @@ -83,6 +83,8 @@ private void generateService(PythonWriter writer) { } } + writer.addDependency(SmithyPythonDependency.SMITHY_CORE); + writer.addImport("smithy_core.retries", "RetryStrategyResolver"); writer.write(""" def __init__(self, config: $1T | None = None, plugins: list[$2T] | None = None): self._config = config or $1T() @@ -95,6 +97,8 @@ def __init__(self, config: $1T | None = None, plugins: list[$2T] | None = None): for plugin in client_plugins: plugin(self._config) + + self._retry_strategy_resolver = RetryStrategyResolver() """, configSymbol, pluginSymbol, writer.consumer(w -> writeDefaultPlugins(w, defaultPlugins))); var topDownIndex = TopDownIndex.of(model); @@ -168,8 +172,7 @@ private void writeSharedOperationInit(PythonWriter writer, OperationShape operat writer.write(""" :param plugins: A list of callables that modify the configuration dynamically. Changes made by these plugins only apply for the duration of the operation - execution and will not affect any other operation invocations. - """); + execution and will not affect any other operation invocations."""); }); @@ -188,6 +191,8 @@ private void writeSharedOperationInit(PythonWriter writer, OperationShape operat writer.addImport("smithy_core.types", "TypedProperties"); writer.addImport("smithy_core.aio.client", "RequestPipeline"); writer.addImport("smithy_core.exceptions", "ExpectationNotMetError"); + writer.addImport("smithy_core.retries", "RetryStrategyOptions"); + writer.addImport("smithy_core.interfaces.retries", "RetryStrategy"); writer.addStdlibImport("copy", "deepcopy"); writer.write(""" @@ -201,6 +206,24 @@ private void writeSharedOperationInit(PythonWriter writer, OperationShape operat plugin(config) if config.protocol is None or config.transport is None: raise ExpectationNotMetError("protocol and transport MUST be set on the config to make calls.") + + # Resolve retry strategy from config + if isinstance(config.retry_strategy, RetryStrategy): + retry_strategy = config.retry_strategy + elif isinstance(config.retry_strategy, RetryStrategyOptions): + retry_strategy = await self._retry_strategy_resolver.resolve_retry_strategy( + options=config.retry_strategy + ) + elif config.retry_strategy is None: + retry_strategy = await self._retry_strategy_resolver.resolve_retry_strategy( + options=RetryStrategyOptions() + ) + else: + raise TypeError( + f"retry_strategy must be RetryStrategy, RetryStrategyOptions, or None, " + f"got {type(config.retry_strategy).__name__}" + ) + pipeline = RequestPipeline( protocol=config.protocol, transport=config.transport @@ -213,7 +236,7 @@ raise ExpectationNotMetError("protocol and transport MUST be set on the config t auth_scheme_resolver=config.auth_scheme_resolver, supported_auth_schemes=config.auth_schemes, endpoint_resolver=config.endpoint_resolver, - retry_strategy=config.retry_strategy, + retry_strategy=retry_strategy, ) """, writer.consumer(w -> writeDefaultPlugins(w, defaultPlugins))); diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/HttpProtocolTestGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/HttpProtocolTestGenerator.java index 83f1258f4..62330fa78 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/HttpProtocolTestGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/HttpProtocolTestGenerator.java @@ -631,6 +631,8 @@ def __init__(self, request: HTTPRequest): class $3L: ""\"An asynchronous HTTP client solely for testing purposes.""\" + TIMEOUT_EXCEPTIONS = () + def __init__(self, *, client_config: HTTPClientConfiguration | None = None): self._client_config = client_config @@ -644,6 +646,8 @@ async def send( class $4L: ""\"An asynchronous HTTP client solely for testing purposes.""\" + TIMEOUT_EXCEPTIONS = () + def __init__( self, *, diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java index 654bea9fe..71fde8b10 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java @@ -22,7 +22,7 @@ public final class SmithyPythonDependency { */ public static final PythonDependency SMITHY_CORE = new PythonDependency( "smithy_core", - "~=0.1.0", + "~=0.2.0", Type.DEPENDENCY, false); @@ -33,7 +33,7 @@ public final class SmithyPythonDependency { */ public static final PythonDependency SMITHY_HTTP = new PythonDependency( "smithy_http", - "~=0.2.0", + "~=0.3.0", Type.DEPENDENCY, false); @@ -60,7 +60,7 @@ public final class SmithyPythonDependency { */ public static final PythonDependency SMITHY_JSON = new PythonDependency( "smithy_json", - "~=0.1.0", + "~=0.2.0", Type.DEPENDENCY, false); @@ -69,7 +69,7 @@ public final class SmithyPythonDependency { */ public static final PythonDependency SMITHY_AWS_EVENT_STREAM = new PythonDependency( "smithy_aws_event_stream", - "~=0.1.0", + "~=0.2.0", Type.DEPENDENCY, false); @@ -78,7 +78,7 @@ public final class SmithyPythonDependency { */ public static final PythonDependency SMITHY_AWS_CORE = new PythonDependency( "smithy_aws_core", - "~=0.1.0", + "~=0.2.0", Type.DEPENDENCY, false); diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ConfigGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ConfigGenerator.java index 4aae8893f..b17847ba9 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ConfigGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ConfigGenerator.java @@ -10,6 +10,7 @@ import java.util.List; import java.util.Locale; import java.util.TreeSet; +import software.amazon.smithy.aws.traits.ServiceTrait; import software.amazon.smithy.codegen.core.Symbol; import software.amazon.smithy.model.knowledge.EventStreamIndex; import software.amazon.smithy.model.knowledge.ServiceIndex; @@ -54,17 +55,20 @@ public final class ConfigGenerator implements Runnable { ConfigProperty.builder() .name("retry_strategy") .type(Symbol.builder() - .name("RetryStrategy") - .namespace("smithy_core.interfaces.retries", ".") - .addDependency(SmithyPythonDependency.SMITHY_CORE) + .name("RetryStrategy | RetryStrategyOptions") + .addReference(Symbol.builder() + .name("RetryStrategy") + .namespace("smithy_core.interfaces.retries", ".") + .addDependency(SmithyPythonDependency.SMITHY_CORE) + .build()) + .addReference(Symbol.builder() + .name("RetryStrategyOptions") + .namespace("smithy_core.retries", ".") + .addDependency(SmithyPythonDependency.SMITHY_CORE) + .build()) .build()) - .documentation("The retry strategy for issuing retry tokens and computing retry delays.") - .nullable(false) - .initialize(writer -> { - writer.addDependency(SmithyPythonDependency.SMITHY_CORE); - writer.addImport("smithy_core.retries", "SimpleRetryStrategy"); - writer.write("self.retry_strategy = retry_strategy or SimpleRetryStrategy()"); - }) + .documentation( + "The retry strategy or options for configuring retry behavior. Can be either a configured RetryStrategy or RetryStrategyOptions to create one.") .build(), ConfigProperty.builder() .name("endpoint_uri") @@ -331,6 +335,11 @@ private void generateConfig(GenerationContext context, PythonWriter writer) { } var finalProperties = List.copyOf(properties); + final String serviceId = context.settings() + .service(context.model()) + .getTrait(ServiceTrait.class) + .map(ServiceTrait::getSdkId) + .orElse(context.settings().service().getName()); writer.pushState(new ConfigSection(finalProperties)); writer.addStdlibImport("dataclasses", "dataclass"); writer.write(""" @@ -345,14 +354,11 @@ def __init__( *, ${C|} ): - \"""Constructor. - ${C|} - \""" ${C|} """, configSymbol.getName(), - context.settings().service().getName(), + serviceId, writer.consumer(w -> writePropertyDeclarations(w, finalProperties)), writer.consumer(w -> writeInitParams(w, finalProperties)), writer.consumer(w -> documentProperties(w, finalProperties)), @@ -376,17 +382,20 @@ private void writeInitParams(PythonWriter writer, Collection pro } private void documentProperties(PythonWriter writer, Collection properties) { - var iter = properties.iterator(); - while (iter.hasNext()) { - var property = iter.next(); - var docs = writer.formatDocs(String.format(":param %s: %s", property.name(), property.documentation())); + writer.writeDocs(() -> { + var iter = properties.iterator(); + writer.write("\nConstructor.\n"); + while (iter.hasNext()) { + var property = iter.next(); + var docs = writer.formatDocs(String.format(":param %s: %s", property.name(), property.documentation())); + + if (iter.hasNext()) { + docs += "\n"; + } - if (iter.hasNext()) { - docs += "\n"; + writer.write(docs); } - - writer.write(docs); - } + }); } private void initializeProperties(PythonWriter writer, Collection properties) { diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/InitGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/InitGenerator.java index 22b7e156d..5c4827e78 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/InitGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/InitGenerator.java @@ -6,6 +6,7 @@ import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Set; import java.util.stream.Collectors; import software.amazon.smithy.python.codegen.GenerationContext; import software.amazon.smithy.utils.SmithyInternalApi; @@ -15,6 +16,8 @@ */ @SmithyInternalApi public final class InitGenerator implements Runnable { + // Set of directories that need __init__.py files + private static final Set PACKAGE_DIRECTORIES = Set.of("src", "tests"); private final GenerationContext context; @@ -31,6 +34,7 @@ public void run() { .stream() .map(Paths::get) .filter(path -> !path.getParent().equals(context.fileManifest().getBaseDir())) + .filter(path -> PACKAGE_DIRECTORIES.contains(path.getName(0).toString())) .collect(Collectors.groupingBy(Path::getParent, Collectors.toSet())); for (var entry : directories.entrySet()) { var initPath = entry.getKey().resolve("__init__.py"); diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/StructureGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/StructureGenerator.java index 1cfa9c2cf..498938f36 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/StructureGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/StructureGenerator.java @@ -242,7 +242,8 @@ private void writeProperties() { ${?useField}\ field(${?sensitive}repr=False, ${/sensitive}${defaultKey:L}=${defaultValue:L})\ ${/useField} - ${?docs}""\"${docs:L}""\"${/docs}""", memberName, symbolProvider.toSymbol(member)); + ${?docs}""\"${docs:L}""\"${/docs} + """, memberName, symbolProvider.toSymbol(member)); writer.popState(); } } diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/PythonWriter.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/PythonWriter.java index 6d3ecac94..5ac42637e 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/PythonWriter.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/PythonWriter.java @@ -127,6 +127,8 @@ public PythonWriter writeDocs(Runnable runnable) { pushState(); writeInline("\"\"\""); runnable.run(); + trimTrailingWhitespaces(); + ensureNewline(); write("\"\"\""); popState(); return this; @@ -143,6 +145,35 @@ public PythonWriter writeDocs(String docs) { return this; } + /** + * Trims all trailing whitespace from the writer buffer. + * + * @return Returns the writer. + */ + public PythonWriter trimTrailingWhitespaces() { + // Disable the writer formatting config to ensure toString() + // returns the unmodified state of the underlying StringBuilder + trimBlankLines(-1); + trimTrailingSpaces(false); + + String current = super.toString(); + int end = current.length() - 1; + while (end >= 0 && Character.isWhitespace(current.charAt(end))) { + end--; + } + + String trailing = current.substring(end + 1); + if (!trailing.isEmpty()) { + unwrite(trailing); + } + + // Re-enable the formatting config + trimBlankLines(); + trimTrailingSpaces(true); + + return this; + } + private static final int MAX_LINE_LENGTH = CodegenUtils.MAX_PREFERRED_LINE_LENGTH - 8; /** diff --git a/codegen/gradle/libs.versions.toml b/codegen/gradle/libs.versions.toml index e28c67c7b..4ccc1b007 100644 --- a/codegen/gradle/libs.versions.toml +++ b/codegen/gradle/libs.versions.toml @@ -1,11 +1,11 @@ [versions] -junit5 = "6.0.0" -smithy = "1.63.0" +junit5 = "6.0.1" +smithy = "1.64.0" test-logger-plugin = "4.0.0" spotbugs = "6.0.22" -spotless = "8.0.0" +spotless = "8.1.0" smithy-gradle-plugins = "1.3.0" -dep-analysis = "3.4.0" +dep-analysis = "3.4.1" jsoup = "1.21.2" commonmark = "0.17.0" diff --git a/packages/smithy-aws-core/.changes/0.2.0.json b/packages/smithy-aws-core/.changes/0.2.0.json new file mode 100644 index 000000000..a6fde5040 --- /dev/null +++ b/packages/smithy-aws-core/.changes/0.2.0.json @@ -0,0 +1,20 @@ +{ + "changes": [ + { + "type": "dependency", + "description": "Bump `smithy-json` from `~=0.1.0` to `~=0.2.0`." + }, + { + "type": "dependency", + "description": "Bump `smithy-core` from `~=0.1.0` to `~=0.2.0`." + }, + { + "type": "dependency", + "description": "Bump `smithy-aws-event-stream` from `~=0.1.0` to `~=0.2.0`." + }, + { + "type": "dependency", + "description": "Bump `smithy-http` from `~=0.2.0` to `~=0.3.0`." + } + ] +} \ No newline at end of file diff --git a/packages/smithy-aws-core/CHANGELOG.md b/packages/smithy-aws-core/CHANGELOG.md index 61400b618..90a18a3d5 100644 --- a/packages/smithy-aws-core/CHANGELOG.md +++ b/packages/smithy-aws-core/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## v0.2.0 + +### Dependencies +* Bump `smithy-json` from `~=0.1.0` to `~=0.2.0`. +* Bump `smithy-core` from `~=0.1.0` to `~=0.2.0`. +* Bump `smithy-aws-event-stream` from `~=0.1.0` to `~=0.2.0`. +* Bump `smithy-http` from `~=0.2.0` to `~=0.3.0`. + ## v0.1.1 ### Dependencies diff --git a/packages/smithy-aws-core/pyproject.toml b/packages/smithy-aws-core/pyproject.toml index a2efc6fe8..c9c2defe5 100644 --- a/packages/smithy-aws-core/pyproject.toml +++ b/packages/smithy-aws-core/pyproject.toml @@ -26,8 +26,8 @@ classifiers = [ "Topic :: Software Development :: Libraries" ] dependencies = [ - "smithy-core~=0.1.0", - "smithy-http~=0.2.0", + "smithy-core~=0.2.0", + "smithy-http~=0.3.0", "aws-sdk-signers~=0.1.0" ] @@ -45,10 +45,10 @@ path = "src/smithy_aws_core/__init__.py" [project.optional-dependencies] eventstream = [ - "smithy-aws-event-stream~=0.1.0" + "smithy-aws-event-stream~=0.2.0" ] json = [ - "smithy-json~=0.1.0" + "smithy-json~=0.2.0" ] [tool.hatch.build] diff --git a/packages/smithy-aws-core/src/smithy_aws_core/__init__.py b/packages/smithy-aws-core/src/smithy_aws_core/__init__.py index aa37e8185..d4291a169 100644 --- a/packages/smithy-aws-core/src/smithy_aws_core/__init__.py +++ b/packages/smithy-aws-core/src/smithy_aws_core/__init__.py @@ -1,4 +1,4 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -__version__ = "0.1.1" +__version__ = "0.2.0" diff --git a/packages/smithy-aws-event-stream/.changes/0.2.0.json b/packages/smithy-aws-event-stream/.changes/0.2.0.json new file mode 100644 index 000000000..829ba710c --- /dev/null +++ b/packages/smithy-aws-event-stream/.changes/0.2.0.json @@ -0,0 +1,8 @@ +{ + "changes": [ + { + "type": "dependency", + "description": "Bump `smithy-core` from `~=0.1.0` to `~=0.2.0`." + } + ] +} \ No newline at end of file diff --git a/packages/smithy-aws-event-stream/CHANGELOG.md b/packages/smithy-aws-event-stream/CHANGELOG.md index 04acd0477..50724dc71 100644 --- a/packages/smithy-aws-event-stream/CHANGELOG.md +++ b/packages/smithy-aws-event-stream/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## v0.2.0 + +### Dependencies +* Bump `smithy-core` from `~=0.1.0` to `~=0.2.0`. + ## v0.1.0 ### Breaking Changes diff --git a/packages/smithy-aws-event-stream/pyproject.toml b/packages/smithy-aws-event-stream/pyproject.toml index 85b231e3e..081b7761b 100644 --- a/packages/smithy-aws-event-stream/pyproject.toml +++ b/packages/smithy-aws-event-stream/pyproject.toml @@ -26,7 +26,7 @@ classifiers = [ "Topic :: Software Development :: Libraries" ] dependencies = [ - "smithy-core~=0.1.0", + "smithy-core~=0.2.0", ] [project.urls] diff --git a/packages/smithy-aws-event-stream/src/smithy_aws_event_stream/__init__.py b/packages/smithy-aws-event-stream/src/smithy_aws_event_stream/__init__.py index 36dc386c5..c392c786c 100644 --- a/packages/smithy-aws-event-stream/src/smithy_aws_event_stream/__init__.py +++ b/packages/smithy-aws-event-stream/src/smithy_aws_event_stream/__init__.py @@ -1,4 +1,4 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -__version__ = "0.1.0" +__version__ = "0.2.0" diff --git a/packages/smithy-aws-event-stream/src/smithy_aws_event_stream/_private/serializers.py b/packages/smithy-aws-event-stream/src/smithy_aws_event_stream/_private/serializers.py index a96d38b5c..63e2b0ac1 100644 --- a/packages/smithy-aws-event-stream/src/smithy_aws_event_stream/_private/serializers.py +++ b/packages/smithy-aws-event-stream/src/smithy_aws_event_stream/_private/serializers.py @@ -59,10 +59,8 @@ def begin_struct(self, schema: "Schema") -> Iterator[ShapeSerializer]: # Note that if we're serializing an operation input or output, it won't be a # union at all, so this won't get triggered. Thankfully, that's what we want. if schema.shape_type is ShapeType.UNION: - try: - yield self - finally: - return + yield self + return headers: dict[str, HEADER_VALUE] = {} diff --git a/packages/smithy-core/.changes/0.2.0.json b/packages/smithy-core/.changes/0.2.0.json new file mode 100644 index 000000000..2d8a52fc3 --- /dev/null +++ b/packages/smithy-core/.changes/0.2.0.json @@ -0,0 +1,8 @@ +{ + "changes": [ + { + "type": "feature", + "description": "Added support for `standard` retry mode." + } + ] +} \ No newline at end of file diff --git a/packages/smithy-core/CHANGELOG.md b/packages/smithy-core/CHANGELOG.md index 52eb36e50..29d49d545 100644 --- a/packages/smithy-core/CHANGELOG.md +++ b/packages/smithy-core/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## v0.2.0 + +### Features +* Added support for `standard` retry mode. + ## v0.1.1 ### Bug fixes diff --git a/packages/smithy-core/src/smithy_core/__init__.py b/packages/smithy-core/src/smithy_core/__init__.py index 2cf04d6ff..f6c4e1583 100644 --- a/packages/smithy-core/src/smithy_core/__init__.py +++ b/packages/smithy-core/src/smithy_core/__init__.py @@ -8,7 +8,7 @@ from . import interfaces, rfc3986 from .exceptions import SmithyError -__version__ = "0.1.1" +__version__ = "0.2.0" class HostType(Enum): diff --git a/packages/smithy-core/src/smithy_core/aio/client.py b/packages/smithy-core/src/smithy_core/aio/client.py index bf27c440c..01f18097b 100644 --- a/packages/smithy-core/src/smithy_core/aio/client.py +++ b/packages/smithy-core/src/smithy_core/aio/client.py @@ -12,7 +12,7 @@ from ..auth import AuthParams from ..deserializers import DeserializeableShape, ShapeDeserializer from ..endpoints import EndpointResolverParams -from ..exceptions import RetryError, SmithyError +from ..exceptions import ClientTimeoutError, RetryError, SmithyError from ..interceptors import ( InputContext, Interceptor, @@ -349,7 +349,7 @@ async def _retry[I: SerializeableShape, O: DeserializeableShape]( if isinstance(output_context.response, Exception): try: - retry_strategy.refresh_retry_token_for_retry( + retry_token = retry_strategy.refresh_retry_token_for_retry( token_to_renew=retry_token, error=output_context.response, ) @@ -448,24 +448,31 @@ async def _handle_attempt[I: SerializeableShape, O: DeserializeableShape]( _LOGGER.debug("Sending request %s", request_context.transport_request) - if request_future is not None: - # If we have an input event stream (or duplex event stream) then we - # need to let the client return ASAP so that it can start sending - # events. So here we start the transport send in a background task - # then set the result of the request future. It's important to sequence - # it just like that so that the client gets a stream that's ready - # to send. - transport_task = asyncio.create_task( - self.transport.send(request=request_context.transport_request) - ) - request_future.set_result(request_context) - transport_response = await transport_task - else: - # If we don't have an input stream, there's no point in creating a - # task, so we just immediately await the coroutine. - transport_response = await self.transport.send( - request=request_context.transport_request - ) + try: + if request_future is not None: + # If we have an input event stream (or duplex event stream) then we + # need to let the client return ASAP so that it can start sending + # events. So here we start the transport send in a background task + # then set the result of the request future. It's important to sequence + # it just like that so that the client gets a stream that's ready + # to send. + transport_task = asyncio.create_task( + self.transport.send(request=request_context.transport_request) + ) + request_future.set_result(request_context) + transport_response = await transport_task + else: + # If we don't have an input stream, there's no point in creating a + # task, so we just immediately await the coroutine. + transport_response = await self.transport.send( + request=request_context.transport_request + ) + except Exception as e: + if isinstance(e, self.transport.TIMEOUT_EXCEPTIONS): + raise ClientTimeoutError( + message=f"Client timeout occurred: {e}" + ) from e + raise _LOGGER.debug("Received response: %s", transport_response) diff --git a/packages/smithy-core/src/smithy_core/aio/interfaces/__init__.py b/packages/smithy-core/src/smithy_core/aio/interfaces/__init__.py index 31d772125..0c900a8cf 100644 --- a/packages/smithy-core/src/smithy_core/aio/interfaces/__init__.py +++ b/packages/smithy-core/src/smithy_core/aio/interfaces/__init__.py @@ -86,7 +86,9 @@ async def resolve_endpoint(self, params: EndpointResolverParams[Any]) -> Endpoin class ClientTransport[I: Request, O: Response](Protocol): - """Protocol-agnostic representation of a client tranport (e.g. an HTTP client).""" + """Protocol-agnostic representation of a client transport (e.g. an HTTP client).""" + + TIMEOUT_EXCEPTIONS: tuple[type[Exception], ...] async def send(self, request: I) -> O: """Send a request over the transport and receive the response.""" diff --git a/packages/smithy-core/src/smithy_core/exceptions.py b/packages/smithy-core/src/smithy_core/exceptions.py index 0e28bd530..0a99976f9 100644 --- a/packages/smithy-core/src/smithy_core/exceptions.py +++ b/packages/smithy-core/src/smithy_core/exceptions.py @@ -50,6 +50,9 @@ class CallError(SmithyError): is_throttling_error: bool = False """Whether the error is a throttling error.""" + is_timeout_error: bool = False + """Whether the error represents a timeout condition.""" + def __post_init__(self): super().__init__(self.message) @@ -61,6 +64,20 @@ class ModeledError(CallError): fault: Fault = "client" +@dataclass(kw_only=True) +class ClientTimeoutError(CallError): + """Exception raised when a client-side timeout occurs. + + This error indicates that the client transport layer encountered a timeout while + attempting to communicate with the server. This typically occurs when network + requests take longer than the configured timeout period. + """ + + fault: Fault = "client" + is_timeout_error: bool = True + is_retry_safe: bool | None = True + + class SerializationError(SmithyError): """Base exception type for exceptions raised during serialization.""" diff --git a/packages/smithy-core/src/smithy_core/interfaces/retries.py b/packages/smithy-core/src/smithy_core/interfaces/retries.py index a5c9d428b..a7f8c4e2d 100644 --- a/packages/smithy-core/src/smithy_core/interfaces/retries.py +++ b/packages/smithy-core/src/smithy_core/interfaces/retries.py @@ -27,6 +27,9 @@ class ErrorRetryInfo(Protocol): is_throttling_error: bool = False """Whether the error is a throttling error.""" + is_timeout_error: bool = False + """Whether the error is a timeout error.""" + class RetryBackoffStrategy(Protocol): """Stateless strategy for computing retry delays based on retry attempt account.""" @@ -52,6 +55,7 @@ class RetryToken(Protocol): """Delay in seconds to wait before the retry attempt.""" +@runtime_checkable class RetryStrategy(Protocol): """Issuer of :py:class:`RetryToken`s.""" @@ -64,7 +68,7 @@ class RetryStrategy(Protocol): def acquire_initial_retry_token( self, *, token_scope: str | None = None ) -> RetryToken: - """Called before any retries (for the first attempt at the operation). + """Create a base retry token for the start of a request. :param token_scope: An arbitrary string accepted by the retry strategy to separate tokens into scopes. diff --git a/packages/smithy-core/src/smithy_core/retries.py b/packages/smithy-core/src/smithy_core/retries.py index 06bf6f988..ce990e6b4 100644 --- a/packages/smithy-core/src/smithy_core/retries.py +++ b/packages/smithy-core/src/smithy_core/retries.py @@ -1,12 +1,62 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 import random +import threading from collections.abc import Callable from dataclasses import dataclass from enum import Enum +from functools import lru_cache +from typing import Any, Literal from .exceptions import RetryError from .interfaces import retries as retries_interface +from .interfaces.retries import RetryStrategy + +RetryStrategyType = Literal["simple", "standard"] + + +@dataclass(kw_only=True, frozen=True) +class RetryStrategyOptions: + """Options for configuring retry behavior.""" + + retry_mode: RetryStrategyType = "standard" + """The retry mode to use.""" + + max_attempts: int | None = None + """Maximum number of attempts (initial attempt plus retries). If None, uses the strategy's default.""" + + +class RetryStrategyResolver: + """Retry strategy resolver that caches retry strategies based on configuration options. + + This resolver caches retry strategy instances based on their configuration to reuse existing + instances of RetryStrategy with the same settings. Uses LRU cache for thread-safe caching. + """ + + async def resolve_retry_strategy( + self, *, options: RetryStrategyOptions + ) -> RetryStrategy: + """Resolve a retry strategy from the provided options, using cache when possible. + + :param options: The retry strategy options to use for creating the strategy. + """ + return self._create_retry_strategy(options.retry_mode, options.max_attempts) + + @lru_cache + def _create_retry_strategy( + self, retry_mode: RetryStrategyType, max_attempts: int | None + ) -> RetryStrategy: + kwargs = {"max_attempts": max_attempts} + filtered_kwargs: dict[str, Any] = { + k: v for k, v in kwargs.items() if v is not None + } + match retry_mode: + case "simple": + return SimpleRetryStrategy(**filtered_kwargs) + case "standard": + return StandardRetryStrategy(**filtered_kwargs) + case _: + raise ValueError(f"Unknown retry mode: {retry_mode}") class ExponentialBackoffJitterType(Enum): @@ -207,7 +257,7 @@ def __init__( def acquire_initial_retry_token( self, *, token_scope: str | None = None ) -> SimpleRetryToken: - """Called before any retries (for the first attempt at the operation). + """Create a base retry token for the start of a request. :param token_scope: This argument is ignored by this retry strategy. """ @@ -242,3 +292,180 @@ def refresh_retry_token_for_retry( def record_success(self, *, token: retries_interface.RetryToken) -> None: """Not used by this retry strategy.""" + + def __deepcopy__(self, memo: Any) -> "SimpleRetryStrategy": + return self + + +class StandardRetryQuota: + """Retry quota used by :py:class:`StandardRetryStrategy`.""" + + INITIAL_RETRY_TOKENS: int = 500 + RETRY_COST: int = 5 + NO_RETRY_INCREMENT: int = 1 + TIMEOUT_RETRY_COST: int = 10 + + def __init__(self, initial_capacity: int = INITIAL_RETRY_TOKENS): + """Initialize retry quota with configurable capacity. + + :param initial_capacity: The initial and maximum capacity for the retry quota. + """ + self._max_capacity = initial_capacity + self._available_capacity = initial_capacity + self._lock = threading.Lock() + + def acquire(self, *, error: Exception) -> int: + """Attempt to acquire capacity for a retry attempt. + + If there's insufficient capacity available, raise an exception. + Otherwise, return the amount of capacity successfully allocated. + """ + + is_timeout = ( + isinstance(error, retries_interface.ErrorRetryInfo) + and error.is_timeout_error + ) + capacity_amount = self.TIMEOUT_RETRY_COST if is_timeout else self.RETRY_COST + + with self._lock: + if capacity_amount > self._available_capacity: + raise RetryError("Retry quota exceeded") + self._available_capacity -= capacity_amount + return capacity_amount + + def release(self, *, release_amount: int) -> None: + """Release capacity back to the retry quota. + + The capacity being released will be truncated if necessary to ensure the max + capacity is never exceeded. + """ + increment = self.NO_RETRY_INCREMENT if release_amount == 0 else release_amount + + if self._available_capacity == self._max_capacity: + return + + with self._lock: + self._available_capacity = min( + self._available_capacity + increment, self._max_capacity + ) + + @property + def available_capacity(self) -> int: + """Return the amount of capacity available.""" + return self._available_capacity + + +@dataclass(kw_only=True) +class StandardRetryToken: + retry_count: int + """Retry count is the total number of attempts minus the initial attempt.""" + + retry_delay: float + """Delay in seconds to wait before the retry attempt.""" + + quota_acquired: int = 0 + """The amount of quota acquired for this retry attempt.""" + + +class StandardRetryStrategy(retries_interface.RetryStrategy): + def __init__( + self, + *, + backoff_strategy: retries_interface.RetryBackoffStrategy | None = None, + max_attempts: int = 3, + retry_quota: StandardRetryQuota | None = None, + ): + """Standard retry strategy using truncated binary exponential backoff with full + jitter. + + :param backoff_strategy: The backoff strategy used by returned tokens to compute + the retry delay. Defaults to :py:class:`ExponentialRetryBackoffStrategy`. + + :param max_attempts: Upper limit on total number of attempts made, including + initial attempt and retries. + + :param retry_quota: The retry quota to use for managing retry capacity. Defaults + to a new :py:class:`StandardRetryQuota` instance. + """ + if max_attempts < 0: + raise ValueError( + f"max_attempts must be a non-negative integer, got {max_attempts}" + ) + + self.backoff_strategy = backoff_strategy or ExponentialRetryBackoffStrategy( + backoff_scale_value=1, + max_backoff=20, + jitter_type=ExponentialBackoffJitterType.FULL, + ) + self.max_attempts = max_attempts + self._retry_quota = retry_quota or StandardRetryQuota() + + def acquire_initial_retry_token( + self, *, token_scope: str | None = None + ) -> StandardRetryToken: + """Create a base retry token for the start of a request. + + :param token_scope: This argument is ignored by this retry strategy. + """ + retry_delay = self.backoff_strategy.compute_next_backoff_delay(0) + return StandardRetryToken(retry_count=0, retry_delay=retry_delay) + + def refresh_retry_token_for_retry( + self, + *, + token_to_renew: retries_interface.RetryToken, + error: Exception, + ) -> StandardRetryToken: + """Replace an existing retry token from a failed attempt with a new token. + + This retry strategy always returns a token until the attempt count stored in + the new token exceeds the ``max_attempts`` value. + + :param token_to_renew: The token used for the previous failed attempt. + :param error: The error that triggered the need for a retry. + :raises RetryError: If no further retry attempts are allowed. + """ + if not isinstance(token_to_renew, StandardRetryToken): + raise TypeError( + f"StandardRetryStrategy requires StandardRetryToken, got {type(token_to_renew).__name__}" + ) + + if isinstance(error, retries_interface.ErrorRetryInfo) and error.is_retry_safe: + retry_count = token_to_renew.retry_count + 1 + if retry_count >= self.max_attempts: + raise RetryError( + f"Reached maximum number of allowed attempts: {self.max_attempts}" + ) from error + + # Acquire additional quota for this retry attempt + # (may raise a RetryError if none is available) + quota_acquired = self._retry_quota.acquire(error=error) + + if error.retry_after is not None: + retry_delay = error.retry_after + else: + retry_delay = self.backoff_strategy.compute_next_backoff_delay( + retry_count + ) + + return StandardRetryToken( + retry_count=retry_count, + retry_delay=retry_delay, + quota_acquired=quota_acquired, + ) + else: + raise RetryError(f"Error is not retryable: {error}") from error + + def record_success(self, *, token: retries_interface.RetryToken) -> None: + """Release retry quota back based on the amount consumed by the last retry. + + :param token: The token used for the previous successful attempt. + """ + if not isinstance(token, StandardRetryToken): + raise TypeError( + f"StandardRetryStrategy requires StandardRetryToken, got {type(token).__name__}" + ) + self._retry_quota.release(release_amount=token.quota_acquired) + + def __deepcopy__(self, memo: Any) -> "StandardRetryStrategy": + return self diff --git a/packages/smithy-core/tests/unit/test_retries.py b/packages/smithy-core/tests/unit/test_retries.py index 0b3c23be4..18f9e380c 100644 --- a/packages/smithy-core/tests/unit/test_retries.py +++ b/packages/smithy-core/tests/unit/test_retries.py @@ -1,10 +1,16 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 - import pytest from smithy_core.exceptions import CallError, RetryError from smithy_core.retries import ExponentialBackoffJitterType as EBJT -from smithy_core.retries import ExponentialRetryBackoffStrategy, SimpleRetryStrategy +from smithy_core.retries import ( + ExponentialRetryBackoffStrategy, + RetryStrategyOptions, + RetryStrategyResolver, + SimpleRetryStrategy, + StandardRetryQuota, + StandardRetryStrategy, +) @pytest.mark.parametrize( @@ -100,3 +106,148 @@ def test_simple_retry_does_not_retry_unsafe() -> None: token = strategy.acquire_initial_retry_token() with pytest.raises(RetryError): strategy.refresh_retry_token_for_retry(token_to_renew=token, error=error) + + +@pytest.mark.parametrize("max_attempts", [2, 3, 10]) +def test_standard_retry_strategy(max_attempts: int) -> None: + strategy = StandardRetryStrategy(max_attempts=max_attempts) + error = CallError(is_retry_safe=True) + token = strategy.acquire_initial_retry_token() + for _ in range(max_attempts - 1): + token = strategy.refresh_retry_token_for_retry( + token_to_renew=token, error=error + ) + with pytest.raises(RetryError): + strategy.refresh_retry_token_for_retry(token_to_renew=token, error=error) + + +@pytest.mark.parametrize( + "error", + [ + Exception(), + CallError(is_retry_safe=None), + CallError(fault="client", is_retry_safe=False), + ], + ids=[ + "unclassified_error", + "safety_unknown_error", + "unsafe_error", + ], +) +def test_standard_retry_does_not_retry(error: Exception | CallError) -> None: + strategy = StandardRetryStrategy() + token = strategy.acquire_initial_retry_token() + with pytest.raises(RetryError): + strategy.refresh_retry_token_for_retry(token_to_renew=token, error=error) + + +def test_standard_retry_after_overrides_backoff() -> None: + strategy = StandardRetryStrategy() + error = CallError(is_retry_safe=True, retry_after=5.5) + token = strategy.acquire_initial_retry_token() + token = strategy.refresh_retry_token_for_retry(token_to_renew=token, error=error) + assert token.retry_delay == 5.5 + + +def test_standard_retry_invalid_max_attempts() -> None: + with pytest.raises(ValueError): + StandardRetryStrategy(max_attempts=-1) + + +@pytest.fixture +def retry_quota() -> StandardRetryQuota: + return StandardRetryQuota(initial_capacity=10) + + +def test_retry_quota_initial_state( + retry_quota: StandardRetryQuota, +) -> None: + assert retry_quota.available_capacity == 10 + + +def test_retry_quota_acquire_success( + retry_quota: StandardRetryQuota, +) -> None: + acquired = retry_quota.acquire(error=Exception()) + assert retry_quota.available_capacity == 10 - acquired + + +def test_retry_quota_acquire_when_exhausted( + retry_quota: StandardRetryQuota, +) -> None: + # Drain capacity until insufficient for next acquire + retry_quota.acquire(error=Exception()) + retry_quota.acquire(error=Exception()) + + # Not enough capacity for another retry (need 5, only 0 left) + with pytest.raises(RetryError, match="Retry quota exceeded"): + retry_quota.acquire(error=Exception()) + + +def test_retry_quota_release_restores_capacity( + retry_quota: StandardRetryQuota, +) -> None: + acquired = retry_quota.acquire(error=Exception()) + retry_quota.release(release_amount=acquired) + assert retry_quota.available_capacity == 10 + + +def test_retry_quota_release_zero_adds_increment( + retry_quota: StandardRetryQuota, +) -> None: + retry_quota.acquire(error=Exception()) + assert retry_quota.available_capacity == 5 + retry_quota.release(release_amount=0) + assert retry_quota.available_capacity == 6 + + +def test_retry_quota_release_caps_at_max( + retry_quota: StandardRetryQuota, +) -> None: + # Drain some capacity + retry_quota.acquire(error=Exception()) + # Release more than we acquired. Should cap at initial capacity. + retry_quota.release(release_amount=50) + assert retry_quota.available_capacity == 10 + + +def test_retry_quota_acquire_timeout_error( + retry_quota: StandardRetryQuota, +) -> None: + timeout_error = CallError(is_timeout_error=True, is_retry_safe=True) + acquired = retry_quota.acquire(error=timeout_error) + assert acquired == StandardRetryQuota.TIMEOUT_RETRY_COST + assert retry_quota.available_capacity == 0 + + +async def test_caching_retry_strategy_default_resolution() -> None: + resolver = RetryStrategyResolver() + options = RetryStrategyOptions() + + strategy = await resolver.resolve_retry_strategy(options=options) + + assert isinstance(strategy, StandardRetryStrategy) + assert strategy.max_attempts == 3 + + +async def test_caching_retry_strategy_resolver_creates_strategies_by_options() -> None: + resolver = RetryStrategyResolver() + + options1 = RetryStrategyOptions(max_attempts=3) + options2 = RetryStrategyOptions(max_attempts=5) + + strategy1 = await resolver.resolve_retry_strategy(options=options1) + strategy2 = await resolver.resolve_retry_strategy(options=options2) + + assert strategy1.max_attempts == 3 + assert strategy2.max_attempts == 5 + + +async def test_caching_retry_strategy_resolver_caches_strategies() -> None: + resolver = RetryStrategyResolver() + + options = RetryStrategyOptions(max_attempts=5) + strategy1 = await resolver.resolve_retry_strategy(options=options) + strategy2 = await resolver.resolve_retry_strategy(options=options) + + assert strategy1 is strategy2 diff --git a/packages/smithy-http/.changes/0.3.0.json b/packages/smithy-http/.changes/0.3.0.json new file mode 100644 index 000000000..6fd7c8d44 --- /dev/null +++ b/packages/smithy-http/.changes/0.3.0.json @@ -0,0 +1,16 @@ +{ + "changes": [ + { + "type": "feature", + "description": "Added `MockHTTPClient` for testing SDK clients without making real HTTP requests." + }, + { + "type": "enhancement", + "description": "Added timeout error detection for HTTP clients" + }, + { + "type": "dependency", + "description": "Bump `smithy-core` from `~=0.1.0` to `~=0.2.0`." + } + ] +} \ No newline at end of file diff --git a/packages/smithy-http/CHANGELOG.md b/packages/smithy-http/CHANGELOG.md index 052989523..e999a0232 100644 --- a/packages/smithy-http/CHANGELOG.md +++ b/packages/smithy-http/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## v0.3.0 + +### Features +* Added `MockHTTPClient` for testing SDK clients without making real HTTP requests. + +### Enhancements +* Added timeout error detection for HTTP clients + +### Dependencies +* Bump `smithy-core` from `~=0.1.0` to `~=0.2.0`. + ## v0.2.1 ### Bug fixes diff --git a/packages/smithy-http/README.md b/packages/smithy-http/README.md index fe7edd994..8b797f423 100644 --- a/packages/smithy-http/README.md +++ b/packages/smithy-http/README.md @@ -1,3 +1,44 @@ # smithy-http This package provides primitives and interfaces for http functionality in tooling generated by Smithy. + +--- + +## Testing + +The `smithy_http.testing` module provides shared utilities for testing HTTP functionality in smithy-python clients. + +### MockHTTPClient + +The `MockHTTPClient` allows you to test smithy-python clients without making actual network calls. It implements the `HTTPClient` interface and provides configurable responses for functional testing. + +#### Basic Usage + +```python +from smithy_http.testing import MockHTTPClient + +# Create mock client and configure responses +mock_client = MockHTTPClient() +mock_client.add_response( + status=200, + headers=[("Content-Type", "application/json")], + body=b'{"message": "success"}' +) + +# Use with your smithy-python client +config = Config(transport=mock_client) +client = TestSmithyServiceClient(config=config) + +# Test your client logic +result = await client.some_operation({"input": "data"}) + +# Inspect what requests were made +assert mock_client.call_count == 1 +captured_request = mock_client.captured_requests[0] +assert result.message == "success" +``` + +### Utilities + +- `create_test_request()`: Helper for creating test HTTPRequest objects +- `MockHTTPClientError`: Exception raised when no responses are queued diff --git a/packages/smithy-http/pyproject.toml b/packages/smithy-http/pyproject.toml index 6060e0d4e..18a5ce0a7 100644 --- a/packages/smithy-http/pyproject.toml +++ b/packages/smithy-http/pyproject.toml @@ -26,7 +26,7 @@ classifiers = [ "Topic :: Software Development :: Libraries" ] dependencies = [ - "smithy-core~=0.1.0", + "smithy-core~=0.2.0", ] [project.urls] diff --git a/packages/smithy-http/src/smithy_http/__init__.py b/packages/smithy-http/src/smithy_http/__init__.py index c4ff55b44..f877b9f44 100644 --- a/packages/smithy-http/src/smithy_http/__init__.py +++ b/packages/smithy-http/src/smithy_http/__init__.py @@ -6,7 +6,7 @@ from . import interfaces from .interfaces import FieldPosition -__version__ = "0.2.1" +__version__ = "0.3.0" class Field(interfaces.Field): diff --git a/packages/smithy-http/src/smithy_http/aio/aiohttp.py b/packages/smithy-http/src/smithy_http/aio/aiohttp.py index 83f4c191f..2d5114726 100644 --- a/packages/smithy-http/src/smithy_http/aio/aiohttp.py +++ b/packages/smithy-http/src/smithy_http/aio/aiohttp.py @@ -52,6 +52,8 @@ def __post_init__(self) -> None: class AIOHTTPClient(HTTPClient): """Implementation of :py:class:`.interfaces.HTTPClient` using aiohttp.""" + TIMEOUT_EXCEPTIONS = (TimeoutError,) + def __init__( self, *, diff --git a/packages/smithy-http/src/smithy_http/aio/crt.py b/packages/smithy-http/src/smithy_http/aio/crt.py index a450ef9c9..db13162e1 100644 --- a/packages/smithy-http/src/smithy_http/aio/crt.py +++ b/packages/smithy-http/src/smithy_http/aio/crt.py @@ -8,6 +8,8 @@ from inspect import iscoroutinefunction from typing import TYPE_CHECKING, Any +from awscrt.exceptions import AwsCrtError + if TYPE_CHECKING: # pyright doesn't like optional imports. This is reasonable because if we use these # in type hints then they'd result in runtime errors. @@ -129,9 +131,16 @@ def __post_init__(self) -> None: _assert_crt() +class _CRTTimeoutError(Exception): + """Internal wrapper for CRT timeout errors.""" + + class AWSCRTHTTPClient(http_aio_interfaces.HTTPClient): _HTTP_PORT = 80 _HTTPS_PORT = 443 + _TIMEOUT_ERROR_NAMES = frozenset(["AWS_IO_SOCKET_TIMEOUT", "AWS_IO_SOCKET_CLOSED"]) + + TIMEOUT_EXCEPTIONS = (_CRTTimeoutError,) def __init__( self, @@ -163,18 +172,23 @@ async def send( :param request: The request including destination URI, fields, payload. :param request_config: Configuration specific to this request. """ - crt_request = self._marshal_request(request) - connection = await self._get_connection(request.destination) + try: + crt_request = self._marshal_request(request) + connection = await self._get_connection(request.destination) - # Convert body to async iterator for request_body_generator - body_generator = self._create_body_generator(request.body) + # Convert body to async iterator for request_body_generator + body_generator = self._create_body_generator(request.body) - crt_stream = connection.request( - crt_request, - request_body_generator=body_generator, - ) + crt_stream = connection.request( + crt_request, + request_body_generator=body_generator, + ) - return await self._await_response(crt_stream) + return await self._await_response(crt_stream) + except AwsCrtError as e: + if e.name in self._TIMEOUT_ERROR_NAMES: + raise _CRTTimeoutError() from e + raise async def _await_response( self, stream: "AIOHttpClientStreamUnified" diff --git a/packages/smithy-http/src/smithy_http/aio/protocols.py b/packages/smithy-http/src/smithy_http/aio/protocols.py index cf25036fe..af32cee16 100644 --- a/packages/smithy-http/src/smithy_http/aio/protocols.py +++ b/packages/smithy-http/src/smithy_http/aio/protocols.py @@ -215,7 +215,6 @@ async def _create_error( ) return error_shape.deserialize(deserializer) - is_throttle = response.status == 429 message = ( f"Unknown error for operation {operation.schema.id} " f"- status: {response.status}" @@ -224,11 +223,17 @@ async def _create_error( message += f" - id: {error_id}" if response.reason is not None: message += f" - reason: {response.status}" + + is_timeout = response.status == 408 + is_throttle = response.status == 429 + fault = "client" if response.status < 500 else "server" + return CallError( message=message, - fault="client" if response.status < 500 else "server", + fault=fault, is_throttling_error=is_throttle, - is_retry_safe=is_throttle or None, + is_timeout_error=is_timeout, + is_retry_safe=is_throttle or is_timeout or None, ) def _matches_content_type(self, response: HTTPResponse) -> bool: diff --git a/packages/smithy-http/src/smithy_http/testing/__init__.py b/packages/smithy-http/src/smithy_http/testing/__init__.py new file mode 100644 index 000000000..960f14adf --- /dev/null +++ b/packages/smithy-http/src/smithy_http/testing/__init__.py @@ -0,0 +1,13 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shared utilities for testing smithy-python clients with an HTTP transport.""" + +from .mockhttp import MockHTTPClient, MockHTTPClientError +from .utils import create_test_request + +__all__ = ( + "MockHTTPClient", + "MockHTTPClientError", + "create_test_request", +) diff --git a/packages/smithy-http/src/smithy_http/testing/mockhttp.py b/packages/smithy-http/src/smithy_http/testing/mockhttp.py new file mode 100644 index 000000000..95d1f758d --- /dev/null +++ b/packages/smithy-http/src/smithy_http/testing/mockhttp.py @@ -0,0 +1,102 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +from collections import deque +from copy import copy +from typing import Any + +from smithy_core.aio.utils import async_list + +from smithy_http import tuples_to_fields +from smithy_http.aio import HTTPResponse +from smithy_http.aio.interfaces import HTTPClient, HTTPRequest +from smithy_http.interfaces import HTTPClientConfiguration, HTTPRequestConfiguration + + +class MockHTTPClient(HTTPClient): + """Implementation of :py:class:`.interfaces.HTTPClient` solely for testing purposes. + + Simulates HTTP request/response behavior. Responses are queued in FIFO order and + requests are captured for inspection. + """ + + TIMEOUT_EXCEPTIONS = (TimeoutError,) + + def __init__( + self, + *, + client_config: HTTPClientConfiguration | None = None, + ) -> None: + """ + :param client_config: Configuration that applies to all requests made with this + client. + """ + self._client_config = client_config + self._response_queue: deque[dict[str, Any]] = deque() + self._captured_requests: list[HTTPRequest] = [] + + def add_response( + self, + status: int = 200, + headers: list[tuple[str, str]] | None = None, + body: bytes = b"", + ) -> None: + """Queue a response for the next request. + + :param status: HTTP status code. + :param headers: HTTP response headers as list of (name, value) tuples. + :param body: Response body as bytes. + """ + self._response_queue.append( + { + "status": status, + "headers": headers or [], + "body": body, + } + ) + + async def send( + self, + request: HTTPRequest, + *, + request_config: HTTPRequestConfiguration | None = None, + ) -> HTTPResponse: + """Send HTTP request and return configured response. + + :param request: The request including destination URI, fields, payload. + :param request_config: Configuration specific to this request. + :returns: Pre-configured HTTP response from the queue. + :raises MockHTTPClientError: If no responses are queued. + """ + self._captured_requests.append(copy(request)) + + # Return next queued response or raise error + if self._response_queue: + response_data = self._response_queue.popleft() + return HTTPResponse( + status=response_data["status"], + fields=tuples_to_fields(response_data["headers"]), + body=async_list([response_data["body"]]), + reason=None, + ) + else: + raise MockHTTPClientError( + "No responses queued in MockHTTPClient. Use add_response() to queue responses." + ) + + @property + def call_count(self) -> int: + """The number of requests made to this client.""" + return len(self._captured_requests) + + @property + def captured_requests(self) -> list[HTTPRequest]: + """The list of all requests captured by this client.""" + return self._captured_requests.copy() + + def __deepcopy__(self, memo: Any) -> "MockHTTPClient": + return self + + +class MockHTTPClientError(Exception): + """Exception raised by MockHTTPClient for test setup issues.""" diff --git a/packages/smithy-http/src/smithy_http/testing/utils.py b/packages/smithy-http/src/smithy_http/testing/utils.py new file mode 100644 index 000000000..9ed3a6476 --- /dev/null +++ b/packages/smithy-http/src/smithy_http/testing/utils.py @@ -0,0 +1,31 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +from smithy_core import URI + +from smithy_http import tuples_to_fields +from smithy_http.aio import HTTPRequest + + +def create_test_request( + method: str = "GET", + host: str = "test.aws.dev", + path: str | None = None, + headers: list[tuple[str, str]] | None = None, + body: bytes = b"", +) -> HTTPRequest: + """Create test HTTPRequest with defaults. + + :param method: HTTP method (GET, POST, etc.) + :param host: Host name (e.g., "test.aws.dev") + :param path: Optional path (e.g., "/users") + :param headers: Optional headers as list of (name, value) tuples + :param body: Request body as bytes + :return: Configured HTTPRequest for testing + """ + return HTTPRequest( + destination=URI(host=host, path=path), + method=method, + fields=tuples_to_fields(headers or []), + body=body, + ) diff --git a/packages/smithy-http/tests/unit/aio/test_protocols.py b/packages/smithy-http/tests/unit/aio/test_protocols.py index ecdb15cfa..4ae18ce67 100644 --- a/packages/smithy-http/tests/unit/aio/test_protocols.py +++ b/packages/smithy-http/tests/unit/aio/test_protocols.py @@ -18,7 +18,7 @@ from smithy_http.aio.protocols import HttpClientProtocol -class TestProtocol(HttpClientProtocol): +class MockProtocol(HttpClientProtocol): _id = ShapeID("ns.foo#bar") @property @@ -125,7 +125,7 @@ def deserialize_response( def test_http_protocol_joins_uris( request_uri: URI, endpoint_uri: URI, expected: URI ) -> None: - protocol = TestProtocol() + protocol = MockProtocol() request = HTTPRequest( destination=request_uri, method="GET", diff --git a/packages/smithy-http/tests/unit/testing/test_mockhttp.py b/packages/smithy-http/tests/unit/testing/test_mockhttp.py new file mode 100644 index 000000000..9efe68589 --- /dev/null +++ b/packages/smithy-http/tests/unit/testing/test_mockhttp.py @@ -0,0 +1,115 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +import pytest +from smithy_http.testing import MockHTTPClient, MockHTTPClientError, create_test_request + + +async def test_default_response(): + # Test error when no responses are queued + mock_client = MockHTTPClient() + request = create_test_request() + + with pytest.raises(MockHTTPClientError, match="No responses queued"): + await mock_client.send(request) + + +async def test_queued_responses_fifo(): + # Test responses are returned in FIFO order + mock_client = MockHTTPClient() + mock_client.add_response(status=404, body=b"not found") + mock_client.add_response(status=500, body=b"server error") + + request = create_test_request() + + response1 = await mock_client.send(request) + assert response1.status == 404 + assert await response1.consume_body_async() == b"not found" + + response2 = await mock_client.send(request) + assert response2.status == 500 + assert await response2.consume_body_async() == b"server error" + + assert mock_client.call_count == 2 + + +async def test_captured_requests(): + # Test all requests are captured for inspection + mock_client = MockHTTPClient() + mock_client.add_response() + mock_client.add_response() + + request1 = create_test_request( + method="GET", + host="test.aws.dev", + ) + request2 = create_test_request( + method="POST", + host="test.aws.dev", + body=b'{"name": "test"}', + ) + + await mock_client.send(request1) + await mock_client.send(request2) + + captured = mock_client.captured_requests + assert len(captured) == 2 + assert captured[0].method == "GET" + assert captured[1].method == "POST" + assert captured[1].body == b'{"name": "test"}' + + +async def test_response_headers(): + # Test response headers are properly set + mock_client = MockHTTPClient() + mock_client.add_response( + status=201, + headers=[ + ("Content-Type", "application/json"), + ("X-Amz-Custom", "test"), + ], + body=b'{"id": 123}', + ) + request = create_test_request() + response = await mock_client.send(request) + + assert response.status == 201 + assert "Content-Type" in response.fields + assert response.fields["Content-Type"].as_string() == "application/json" + assert "X-Amz-Custom" in response.fields + assert response.fields["X-Amz-Custom"].as_string() == "test" + + +async def test_call_count_tracking(): + # Test call count is tracked correctly + mock_client = MockHTTPClient() + mock_client.add_response() + mock_client.add_response() + + request = create_test_request() + + assert mock_client.call_count == 0 + + await mock_client.send(request) + assert mock_client.call_count == 1 + + await mock_client.send(request) + assert mock_client.call_count == 2 + + +async def test_captured_requests_copy(): + # Test that captured_requests returns a copy to prevent modifications + mock_client = MockHTTPClient() + mock_client.add_response() + + request = create_test_request() + + await mock_client.send(request) + + captured1 = mock_client.captured_requests + captured2 = mock_client.captured_requests + + # Should be different list objects + assert captured1 is not captured2 + # But with same content + assert len(captured1) == len(captured2) == 1 diff --git a/packages/smithy-http/tests/unit/testing/test_utils.py b/packages/smithy-http/tests/unit/testing/test_utils.py new file mode 100644 index 000000000..65bcbf75d --- /dev/null +++ b/packages/smithy-http/tests/unit/testing/test_utils.py @@ -0,0 +1,42 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +from smithy_http.testing import create_test_request + + +def test_create_test_request_defaults(): + request = create_test_request() + + assert request.method == "GET" + assert request.destination.host == "test.aws.dev" + assert request.destination.path is None + assert request.body == b"" + assert len(request.fields) == 0 + + +def test_create_test_request_custom_values(): + request = create_test_request( + method="POST", + host="api.example.com", + path="/users", + headers=[ + ("Content-Type", "application/json"), + ("Authorization", "AWS4-HMAC-SHA256"), + ], + body=b'{"name": "test"}', + ) + + assert request.method == "POST" + assert request.destination.host == "api.example.com" + assert request.destination.path == "/users" + assert request.body == b'{"name": "test"}' + + assert "Content-Type" in request.fields + assert request.fields["Content-Type"].as_string() == "application/json" + assert "Authorization" in request.fields + assert request.fields["Authorization"].as_string() == "AWS4-HMAC-SHA256" + + +def test_create_test_request_empty_headers(): + request = create_test_request(headers=[]) + assert len(request.fields) == 0 diff --git a/packages/smithy-json/.changes/0.2.0.json b/packages/smithy-json/.changes/0.2.0.json new file mode 100644 index 000000000..829ba710c --- /dev/null +++ b/packages/smithy-json/.changes/0.2.0.json @@ -0,0 +1,8 @@ +{ + "changes": [ + { + "type": "dependency", + "description": "Bump `smithy-core` from `~=0.1.0` to `~=0.2.0`." + } + ] +} \ No newline at end of file diff --git a/packages/smithy-json/CHANGELOG.md b/packages/smithy-json/CHANGELOG.md index 7f6e786b5..d1f01ed00 100644 --- a/packages/smithy-json/CHANGELOG.md +++ b/packages/smithy-json/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## v0.2.0 + +### Dependencies +* Bump `smithy-core` from `~=0.1.0` to `~=0.2.0`. + ## v0.1.0 ### Enhancements diff --git a/packages/smithy-json/pyproject.toml b/packages/smithy-json/pyproject.toml index f62f7cd00..d6ebe1452 100644 --- a/packages/smithy-json/pyproject.toml +++ b/packages/smithy-json/pyproject.toml @@ -27,7 +27,7 @@ classifiers = [ ] dependencies = [ "ijson>=3.3.0", - "smithy-core~=0.1.0", + "smithy-core~=0.2.0", ] [project.urls] diff --git a/packages/smithy-json/src/smithy_json/__init__.py b/packages/smithy-json/src/smithy_json/__init__.py index d5a412b5f..a16b4e4b4 100644 --- a/packages/smithy-json/src/smithy_json/__init__.py +++ b/packages/smithy-json/src/smithy_json/__init__.py @@ -13,7 +13,7 @@ from ._private.serializers import JSONShapeSerializer as _JSONShapeSerializer from .settings import JSONSettings -__version__ = "0.1.0" +__version__ = "0.2.0" __all__ = ("JSONCodec", "JSONDocument", "JSONSettings")