This repository was archived by the owner on May 21, 2026. It is now read-only.
Repository navigation
Expand file tree
/
Copy pathweb_scrape.py
More file actions
72 lines (62 loc) · 2.3 KB
/
Copy pathweb_scrape.py
File metadata and controls
72 lines (62 loc) · 2.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
"""
title: Jina AI Web Scrape
author: OVINC CN
git_url: https://github.com/OVINC-CN/OpenWebUIPlugin.git
description: Using Jina AI for Web Scrape
requirements: httpx
version: 0.0.1
licence: MIT
"""
import logging
import traceback
from urllib.parse import quote
from httpx import AsyncClient
from pydantic import BaseModel, Field
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
class Tools:
class Valves(BaseModel):
scrape_proxy: str = Field(default="", description="proxy for scrape")
timeout: int = Field(default=60, description="timeout waiting for scraping result")
def __init__(self):
self.valves = self.Valves()
async def web_scrape(self, url: str, __event_emitter__: callable, __metadata__: dict, __user__: dict) -> str:
"""
Scrape and process a web page
:param url: The URL to be scraped.
:return: The scraped and processed content without the Links/Buttons section, or an error message.
"""
logger.info("[web_scrape] %s %s %s", __user__.get("id"), __metadata__.get("chat_id"), url)
await __event_emitter__(
{
"type": "status",
"data": {"description": f"scraping {url}", "done": False, "hidden": False},
}
)
client = AsyncClient(
proxy=self.valves.scrape_proxy or None,
headers={"X-No-Cache": "true", "X-With-Images-Summary": "true", "X-With-Links-Summary": "true"},
timeout=self.valves.timeout,
)
try:
response = await client.get(url=f"https://r.jina.ai/{quote(url)}")
response.raise_for_status()
await __event_emitter__(
{
"type": "status",
"data": {"description": f"scrap success ({url})", "done": True, "hidden": False},
}
)
return response.text
except Exception as err:
message = f"failed to scrap {err}"
print(f"{message}\n{traceback.format_exc()}")
await __event_emitter__(
{
"type": "status",
"data": {"description": message, "done": True, "hidden": True},
}
)
return message
finally:
await client.aclose()