From 7cd9dc878f63f09e95f88c584dafa4b9046f284a Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Fri, 30 May 2025 09:24:07 -0400 Subject: [PATCH] Add info on garbage collection of async clients --- README.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b83cb47c74..3c38e3e135 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,26 @@ async def main() -> None: asyncio.run(main()) ``` -Functionality between the synchronous and asynchronous clients is otherwise identical. +Functionality between the synchronous and asynchronous clients is otherwise mostly identical. + +### Async Garbage Collection + +When opening multiple `AsyncOpenAI` clients, it is recommended to use `with` blocks to handle +resource management instead of relying on traditional garbage collection. Failing to do this +may result in leaked connections and degraded performance over time since the async event loop +can be cleaned up before the `AsyncOpenAI` client has a chance to clean up its resources. + +``` +async def callModel(prompt) -> None: + async with getAsyncVLLMClient(api_key=os.environ.get("OPENAI_API_KEY")) as client: + response = await client.responses.create( + model="gpt-4o", input=prompt + ) + +prompts = [...] +for prompt in prompts: + asyncio.run(callModel(prompt)) +``` ## Streaming responses