Merge pull request #2768 from hlohaus/27Feb

Add ToolSupportProvider

docs/pydantic_ai.md

@@ -129,6 +129,73 @@ This example demonstrates the use of a custom Pydantic model (`MyModel`) to capt
 
 ---
 
+### Support for Models/Providers without Tool Call Support
+
+For models/providers that do not fully support tool calls or lack a direct API for structured output, the `ToolSupportProvider` can be used to bridge the gap. This provider ensures that the agent properly formats the response, even when the model itself doesn't have built-in support for structured outputs. It does so by leveraging a tool list and creating a response format when only one tool is used.
+
+### Example for Models/Providers without Tool Support (Single Tool Usage)
+
+```python
+from pydantic import BaseModel
+from pydantic_ai import Agent
+from pydantic_ai.models import ModelSettings
+from g4f.integration.pydantic_ai import AIModel
+from g4f.providers.tool_support import ToolSupportProvider
+
+from g4f import debug
+debug.logging = True
+
+# Define a custom model for structured output (e.g., city and country)
+class MyModel(BaseModel):
+    city: str
+    country: str
+
+# Create the agent for a model with tool support (using one tool)
+agent = Agent(AIModel(
+    "PollinationsAI:openai", # Specify the provider and model
+    ToolSupportProvider # Use ToolSupportProvider to handle tool-based response formatting
+), result_type=MyModel, model_settings=ModelSettings(temperature=0))
+
+if __name__ == '__main__':
+    # Run the agent with a query to extract information (e.g., city and country)
+    result = agent.run_sync('European city with the bear.')
+    print(result.data)  # Structured output of city and country
+    print(result.usage()) # Usage statistics
+```
+
+### Explanation:
+
+- **`ToolSupportProvider` as a Bridge:** The `ToolSupportProvider` acts as a bridge between the agent and the model, ensuring that the response is formatted into a structured output, even if the model doesn't have an API that directly supports such formatting.
+
+  - For instance, if the model generates raw text or unstructured data, the `ToolSupportProvider` will convert this into the expected format (like `MyModel`), allowing the agent to process it as structured data.
+
+- **Model Initialization:** We initialize the agent with the `PollinationsAI:openai` model, which may not have a built-in API for returning structured outputs. Instead, it relies on the `ToolSupportProvider` to format the output.
+
+- **Custom Result Model:** We define a custom Pydantic model (`MyModel`) to capture the expected output in a structured way (e.g., `city` and `country` fields). This helps ensure that even when the model doesn't support structured data, the agent can interpret and format it.
+
+- **Debug Logging:** The `g4f.debug.logging` is enabled to provide detailed logs for troubleshooting and monitoring the agent's execution.
+
+### Example Output:
+
+```bash
+city='Berlin'
+country='Germany'
+usage={'prompt_tokens': 15, 'completion_tokens': 50}
+```
+
+### Key Points:
+
+- **`ToolSupportProvider` Role:** The `ToolSupportProvider` ensures that the agent formats the raw or unstructured response from the model into a structured format, even if the model itself lacks built-in support for structured data.
+
+- **Single Tool Usage:** The `ToolSupportProvider` is particularly useful when only one tool is used by the model, and it needs to format or transform the model's output into a structured response without additional tools.
+
+### Notes:
+
+- This approach is ideal for models that return unstructured text or data that needs to be transformed into a structured format (e.g., Pydantic models).
+- The `ToolSupportProvider` bridges the gap between the model's output and the expected structured format, enabling seamless integration into workflows that require structured responses.
+
+---
+
 ## LangChain Integration Example
 
 For users working with LangChain, here is an example demonstrating how to integrate G4F models into a LangChain environment:

g4f/Provider/PollinationsAI.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import json
 import random
 import requests
 from urllib.parse import quote_plus
@@ -15,6 +14,7 @@
 from ..requests.raise_for_status import raise_for_status
 from ..requests.aiohttp import get_connector
 from ..providers.response import ImageResponse, ImagePreview, FinishReason, Usage
+from .. import debug
 
 DEFAULT_HEADERS = {
     'Accept': '*/*',
@@ -74,9 +74,11 @@ def get_models(cls, **kwargs):
             try:
                 # Update of image models
                 image_response = requests.get("https://image.pollinations.ai/models")
-                image_response.raise_for_status()
-                new_image_models = image_response.json()
-
+                if image_response.ok:
+                    new_image_models = image_response.json()
+                else:
+                    new_image_models = []
+
                 # Combine models without duplicates
                 all_image_models = (
                     cls.image_models +  # Already contains the default
@@ -112,8 +114,8 @@ def get_models(cls, **kwargs):
                     cls.text_models = [cls.default_model]
                 if not cls.image_models:
                     cls.image_models = [cls.default_image_model]
-                raise RuntimeError(f"Failed to fetch models: {e}") from e
-            
+                debug.error(f"Failed to fetch models: {e}")
+
         return cls.text_models + cls.image_models
 
     @classmethod

g4f/Provider/hf/HuggingFaceAPI.py

@@ -61,10 +61,10 @@ async def create_async_generator(
         images: ImagesType = None,
         **kwargs
     ):
-        if model in cls.model_aliases:
-            model = cls.model_aliases[model]
         if model == llama_models["name"]:
             model = llama_models["text"] if images is None else llama_models["vision"]
+        if model in cls.model_aliases:
+            model = cls.model_aliases[model]
         api_base = f"https://api-inference.huggingface.co/models/{model}/v1"
         pipeline_tag = await cls.get_pipline_tag(model, api_key)
         if pipeline_tag not in ("text-generation", "image-text-to-text"):

g4f/Provider/hf/models.py

@@ -20,6 +20,7 @@
 model_aliases = {
     ### Chat ###
     "qwen-2.5-72b": "Qwen/Qwen2.5-Coder-32B-Instruct",
+    "llama-3": "meta-llama/Llama-3.3-70B-Instruct",
     "llama-3.3-70b": "meta-llama/Llama-3.3-70B-Instruct",
     "command-r-plus": "CohereForAI/c4ai-command-r-plus-08-2024",
     "deepseek-r1": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B",

g4f/Provider/template/OpenaiTemplate.py

@@ -145,7 +145,6 @@ async def create_async_generator(
                 elif content_type.startswith("text/event-stream"):
                     await raise_for_status(response)
                     first = True
-                    is_thinking = 0
                     async for line in response.iter_lines():
                         if line.startswith(b"data: "):
                             chunk = line[6:]

g4f/client/__init__.py

@@ -306,8 +306,8 @@ def create(
 
         response = iter_run_tools(
             provider.get_create_function(),
-            model,
-            messages,
+            model=model,
+            messages=messages,
             stream=stream,
             **filter_none(
                 proxy=self.client.proxy if proxy is None else proxy,
@@ -592,8 +592,8 @@ def create(
 
         response = async_iter_run_tools(
             provider,
-            model,
-            messages,
+            model=model,
+            messages=messages,
             stream=stream,
             **filter_none(
                 proxy=self.client.proxy if proxy is None else proxy,

g4f/debug.py

@@ -1,10 +1,7 @@
 import sys
-from .providers.types import ProviderType
 
 logging: bool = False
 version_check: bool = True
-last_provider: ProviderType = None
-last_model: str = None
 version: str = None
 log_handler: callable = print
 logs: list = []
@@ -14,4 +11,7 @@ def log(text, file = None):
         log_handler(text, file=file)
 
 def error(error, name: str = None):
-    log(error if isinstance(error, str) else f"{type(error).__name__ if name is None else name}: {error}", file=sys.stderr)
+    log(
+        error if isinstance(error, str) else f"{type(error).__name__ if name is None else name}: {error}",
+        file=sys.stderr
+    )

g4f/gui/client/demo.html

@@ -201,7 +201,7 @@
 </head>
 <body>
     <iframe id="background"></iframe>
-    <img id="image-feed" alt="Image Feed">
+    <img id="image-feed" class="hidden" alt="Image Feed">
 
     <!-- Gradient Background Circle -->
     <div class="gradient"></div>
@@ -336,14 +336,15 @@
             const images = []
             eventSource.onmessage = (event) => {
                 const data = JSON.parse(event.data);
-                if (data.nsfw || !data.nologo || data.width < 1024 || !data.imageURL) {
+                if (data.nsfw || !data.nologo || data.width < 1024 || !data.imageURL || data.isChild) {
                     return;
                 }
                 const lower = data.prompt.toLowerCase();
-                const tags = ["logo", "infographic", "warts","prostitute", "curvy", "breasts", "written", "bodies", "naked", "classroom", "malone", "dirty", "shoes", "shower", "banner", "fat", "nipples", "couple", "sexual", "sandal", "supplier", "overlord", "succubus", "platinum", "cracy", "crazy", "lamic", "ropes", "cables", "wires", "dirty", "messy", "cluttered", "chaotic", "disorganized", "disorderly", "untidy", "unorganized", "unorderly", "unsystematic", "disarranged", "disarrayed", "disheveled", "disordered", "jumbled", "muddled", "scattered", "shambolic", "sloppy", "unkept", "unruly"];
+                const tags = ["nsfw", "timeline", "soap", "orally", "heel", "latex", "bathroom", "boobs", "charts", " text ", "gel", "logo", "infographic", "warts", " bra ", "prostitute", "curvy", "breasts", "written", "bodies", "naked", "classroom", "malone", "dirty", "shoes", "shower", "banner", "fat", "nipples", "couple", "sexual", "sandal", "supplier", "overlord", "succubus", "platinum", "cracy", "crazy", "lamic", "ropes", "cables", "wires", "dirty", "messy", "cluttered", "chaotic", "disorganized", "disorderly", "untidy", "unorganized", "unorderly", "unsystematic", "disarranged", "disarrayed", "disheveled", "disordered", "jumbled", "muddled", "scattered", "shambolic", "sloppy", "unkept", "unruly"];
                 for (i in tags) {
                     if (lower.indexOf(tags[i]) != -1) {
                         console.log("Skipping image with tag: " + tags[i]);
+                        console.debug("Skipping image:", data.imageURL);
                         return;
                     }
                 }
@@ -354,7 +355,10 @@
             }
             setInterval(() => {
                 if (images.length > 0) {
+                    imageFeed.classList.remove("hidden");
                     imageFeed.src = images.shift();
+                } else if(imageFeed) {
+                    imageFeed.remove();
                 }
             }, 7000);
         })();

g4f/providers/tool_support.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+import json
+
+from ..typing import AsyncResult, Messages, ImagesType
+from ..providers.asyncio import to_async_iterator
+from ..client.service import get_model_and_provider
+from ..client.helper import filter_json
+from .base_provider import AsyncGeneratorProvider
+from .response import ToolCalls, FinishReason
+
+class ToolSupportProvider(AsyncGeneratorProvider):
+    working = True
+
+    @classmethod
+    async def create_async_generator(
+        cls,
+        model: str,
+        messages: Messages,
+        stream: bool = True,
+        images: ImagesType = None,
+        tools: list[str] = None,
+        response_format: dict = None,
+        **kwargs
+    ) -> AsyncResult:
+        provider = None
+        if ":" in model:
+            provider, model = model.split(":", 1)
+        model, provider = get_model_and_provider(
+            model, provider,
+            stream, logging=False,
+            has_images=images is not None
+        )
+        if tools is not None:
+            if len(tools) > 1:
+                raise ValueError("Only one tool is supported.")
+            if response_format is None:
+                response_format = {"type": "json"}
+            tools = tools.pop()
+            lines = ["Respone in JSON format."]
+            properties = tools["function"]["parameters"]["properties"]
+            properties = {key: value["type"] for key, value in properties.items()}
+            lines.append(f"Response format: {json.dumps(properties, indent=2)}")
+            messages = [{"role": "user", "content": "\n".join(lines)}] + messages
+
+        finish = None
+        chunks = []
+        async for chunk in provider.get_async_create_function()(
+            model,
+            messages,
+            stream=stream,
+            images=images,
+            response_format=response_format,
+            **kwargs
+        ):
+            if isinstance(chunk, FinishReason):
+                finish = chunk
+                break
+            elif isinstance(chunk, str):
+                chunks.append(chunk)
+            else:
+                yield chunk
+
+        chunks = "".join(chunks)
+        if tools is not None:
+            yield ToolCalls([{
+                "id": "",
+                "type": "function",
+                "function": {
+                    "name": tools["function"]["name"],
+                    "arguments": filter_json(chunks)
+                }
+            }])
+        yield chunks
+        if finish is not None:
+            yield finish