8
8
"""
9
9
10
10
import os
11
- import sys
12
- import logging
13
- import traceback
14
11
from typing import Any , Dict
15
12
16
13
import httpx
17
14
from mcp .server .fastmcp import FastMCP
18
15
19
16
20
- # Configure logging
21
- logging .basicConfig (
22
- level = logging .INFO ,
23
- format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' ,
24
- handlers = [
25
- logging .StreamHandler (sys .stderr )
26
- ]
27
- )
28
- logger = logging .getLogger ("scrapegraph_mcp" )
29
-
30
-
31
17
class ScapeGraphClient :
32
18
"""Client for interacting with the ScapeGraph API."""
33
19
@@ -40,14 +26,12 @@ def __init__(self, api_key: str):
40
26
Args:
41
27
api_key: API key for ScapeGraph API
42
28
"""
43
- logger .info ("Initializing ScapeGraphClient" )
44
29
self .api_key = api_key
45
30
self .headers = {
46
31
"SGAI-APIKEY" : api_key ,
47
32
"Content-Type" : "application/json"
48
33
}
49
34
self .client = httpx .Client (timeout = 60.0 )
50
- logger .info ("ScapeGraphClient initialized successfully" )
51
35
52
36
def markdownify (self , website_url : str ) -> Dict [str , Any ]:
53
37
"""
@@ -59,26 +43,18 @@ def markdownify(self, website_url: str) -> Dict[str, Any]:
59
43
Returns:
60
44
Dictionary containing the markdown result
61
45
"""
62
- logger .info (f"Calling markdownify for URL: { website_url } " )
63
46
url = f"{ self .BASE_URL } /markdownify"
64
47
data = {
65
48
"website_url" : website_url
66
49
}
67
50
68
- try :
69
- logger .debug (f"Making POST request to { url } " )
70
- response = self .client .post (url , headers = self .headers , json = data )
71
-
72
- if response .status_code != 200 :
73
- error_msg = f"Error { response .status_code } : { response .text } "
74
- logger .error (f"API request failed: { error_msg } " )
75
- raise Exception (error_msg )
76
-
77
- logger .info ("markdownify request successful" )
78
- return response .json ()
79
- except Exception as e :
80
- logger .error (f"Exception in markdownify: { str (e )} " )
81
- raise
51
+ response = self .client .post (url , headers = self .headers , json = data )
52
+
53
+ if response .status_code != 200 :
54
+ error_msg = f"Error { response .status_code } : { response .text } "
55
+ raise Exception (error_msg )
56
+
57
+ return response .json ()
82
58
83
59
def smartscraper (self , user_prompt : str , website_url : str ) -> Dict [str , Any ]:
84
60
"""
@@ -91,27 +67,19 @@ def smartscraper(self, user_prompt: str, website_url: str) -> Dict[str, Any]:
91
67
Returns:
92
68
Dictionary containing the extracted data
93
69
"""
94
- logger .info (f"Calling smartscraper for URL: { website_url } with prompt: { user_prompt } " )
95
70
url = f"{ self .BASE_URL } /smartscraper"
96
71
data = {
97
72
"user_prompt" : user_prompt ,
98
73
"website_url" : website_url
99
74
}
100
75
101
- try :
102
- logger .debug (f"Making POST request to { url } " )
103
- response = self .client .post (url , headers = self .headers , json = data )
104
-
105
- if response .status_code != 200 :
106
- error_msg = f"Error { response .status_code } : { response .text } "
107
- logger .error (f"API request failed: { error_msg } " )
108
- raise Exception (error_msg )
109
-
110
- logger .info ("smartscraper request successful" )
111
- return response .json ()
112
- except Exception as e :
113
- logger .error (f"Exception in smartscraper: { str (e )} " )
114
- raise
76
+ response = self .client .post (url , headers = self .headers , json = data )
77
+
78
+ if response .status_code != 200 :
79
+ error_msg = f"Error { response .status_code } : { response .text } "
80
+ raise Exception (error_msg )
81
+
82
+ return response .json ()
115
83
116
84
def searchscraper (self , user_prompt : str ) -> Dict [str , Any ]:
117
85
"""
@@ -123,57 +91,30 @@ def searchscraper(self, user_prompt: str) -> Dict[str, Any]:
123
91
Returns:
124
92
Dictionary containing search results and reference URLs
125
93
"""
126
- logger .info (f"Calling searchscraper with prompt: { user_prompt } " )
127
94
url = f"{ self .BASE_URL } /searchscraper"
128
95
data = {
129
96
"user_prompt" : user_prompt
130
97
}
131
98
132
- try :
133
- logger .debug (f"Making POST request to { url } " )
134
- response = self .client .post (url , headers = self .headers , json = data )
135
-
136
- if response .status_code != 200 :
137
- error_msg = f"Error { response .status_code } : { response .text } "
138
- logger .error (f"API request failed: { error_msg } " )
139
- raise Exception (error_msg )
140
-
141
- logger .info ("searchscraper request successful" )
142
- return response .json ()
143
- except Exception as e :
144
- logger .error (f"Exception in searchscraper: { str (e )} " )
145
- raise
99
+ response = self .client .post (url , headers = self .headers , json = data )
100
+
101
+ if response .status_code != 200 :
102
+ error_msg = f"Error { response .status_code } : { response .text } "
103
+ raise Exception (error_msg )
104
+
105
+ return response .json ()
146
106
147
107
def close (self ) -> None :
148
108
"""Close the HTTP client."""
149
- logger .info ("Closing ScapeGraphClient" )
150
109
self .client .close ()
151
- logger .info ("ScapeGraphClient closed" )
152
-
153
110
154
- # Log environment information
155
- logger .info (f"Python version: { sys .version } " )
156
- logger .info (f"Current working directory: { os .getcwd ()} " )
157
- logger .info (f"PATH environment variable: { os .environ .get ('PATH' , 'Not set' )} " )
158
111
159
112
# Create MCP server
160
- logger .info ("Creating MCP server" )
161
113
mcp = FastMCP ("ScapeGraph API MCP Server" )
162
- logger .info ("MCP server created" )
163
114
164
115
# Default API key (will be overridden in main or by direct assignment)
165
116
default_api_key = os .environ .get ("SGAI_API_KEY" )
166
- logger .info (f"SGAI_API_KEY environment variable is { 'set' if default_api_key else 'not set' } " )
167
-
168
- scrapegraph_client = None
169
- if default_api_key :
170
- try :
171
- logger .info ("Initializing ScapeGraphClient with default API key" )
172
- scrapegraph_client = ScapeGraphClient (default_api_key )
173
- logger .info ("ScapeGraphClient initialized successfully" )
174
- except Exception as e :
175
- logger .error (f"Failed to initialize ScapeGraphClient: { str (e )} " )
176
- logger .error (traceback .format_exc ())
117
+ scrapegraph_client = ScapeGraphClient (default_api_key ) if default_api_key else None
177
118
178
119
179
120
# Add tool for markdownify
@@ -188,19 +129,12 @@ def markdownify(website_url: str) -> Dict[str, Any]:
188
129
Returns:
189
130
Dictionary containing the markdown result
190
131
"""
191
- logger .info (f"Tool markdownify called with URL: { website_url } " )
192
-
193
132
if scrapegraph_client is None :
194
- logger .warning ("ScapeGraph client not initialized" )
195
133
return {"error" : "ScapeGraph client not initialized. Please provide an API key." }
196
134
197
135
try :
198
- result = scrapegraph_client .markdownify (website_url )
199
- logger .info ("markdownify tool call successful" )
200
- return result
136
+ return scrapegraph_client .markdownify (website_url )
201
137
except Exception as e :
202
- logger .error (f"Error in markdownify tool: { str (e )} " )
203
- logger .error (traceback .format_exc ())
204
138
return {"error" : str (e )}
205
139
206
140
@@ -220,19 +154,12 @@ def smartscraper(
220
154
Returns:
221
155
Dictionary containing the extracted data
222
156
"""
223
- logger .info (f"Tool smartscraper called with URL: { website_url } and prompt: { user_prompt } " )
224
-
225
157
if scrapegraph_client is None :
226
- logger .warning ("ScapeGraph client not initialized" )
227
158
return {"error" : "ScapeGraph client not initialized. Please provide an API key." }
228
159
229
160
try :
230
- result = scrapegraph_client .smartscraper (user_prompt , website_url )
231
- logger .info ("smartscraper tool call successful" )
232
- return result
161
+ return scrapegraph_client .smartscraper (user_prompt , website_url )
233
162
except Exception as e :
234
- logger .error (f"Error in smartscraper tool: { str (e )} " )
235
- logger .error (traceback .format_exc ())
236
163
return {"error" : str (e )}
237
164
238
165
@@ -250,40 +177,20 @@ def searchscraper(
250
177
Returns:
251
178
Dictionary containing search results and reference URLs
252
179
"""
253
- logger .info (f"Tool searchscraper called with prompt: { user_prompt } " )
254
-
255
180
if scrapegraph_client is None :
256
- logger .warning ("ScapeGraph client not initialized" )
257
181
return {"error" : "ScapeGraph client not initialized. Please provide an API key." }
258
182
259
183
try :
260
- result = scrapegraph_client .searchscraper (user_prompt )
261
- logger .info ("searchscraper tool call successful" )
262
- return result
184
+ return scrapegraph_client .searchscraper (user_prompt )
263
185
except Exception as e :
264
- logger .error (f"Error in searchscraper tool: { str (e )} " )
265
- logger .error (traceback .format_exc ())
266
186
return {"error" : str (e )}
267
187
268
188
269
189
def main () -> None :
270
190
"""Run the ScapeGraph MCP server."""
271
- try :
272
- logger .info ("Starting ScapeGraph MCP server!" )
273
- print ("Starting ScapeGraph MCP server!" , file = sys .stderr )
274
-
275
- # Log system information
276
- logger .info (f"Python executable: { sys .executable } " )
277
- logger .info (f"Arguments: { sys .argv } " )
278
-
279
- # Run the server
280
- logger .info ("Running MCP server with stdio transport" )
281
- mcp .run (transport = "stdio" )
282
- except Exception as e :
283
- logger .critical (f"Fatal error in main: { str (e )} " )
284
- logger .critical (traceback .format_exc ())
285
- print (f"Fatal error: { str (e )} " , file = sys .stderr )
286
- sys .exit (1 )
191
+ print ("Starting ScapeGraph MCP server!" )
192
+ # Run the server
193
+ mcp .run (transport = "stdio" )
287
194
288
195
289
196
if __name__ == "__main__" :
0 commit comments