feat: implemented search scraper functionality

Author: DPende

scrapegraph-js/examples/getSearchScraperRequest_example.js

@@ -0,0 +1,12 @@
+import { getSearchScraperRequest } from 'scrapegraph-js';
+import 'dotenv/config';
+
+const apiKey = process.env.SGAI_APIKEY;
+const requestId = '64801288-6e3b-41f3-9d94-07cff3829e15';
+
+try {
+  const requestInfo = await getSearchScraperRequest(apiKey, requestId);
+  console.log(requestInfo);
+} catch (error) {
+  console.error(error);
+}

scrapegraph-js/examples/schema_searchScraper_example.js

@@ -0,0 +1,19 @@
+import { searchScraper } from 'scrapegraph-js';
+import { z } from 'zod';
+import 'dotenv/config';
+
+const apiKey = process.env.SGAI_APIKEY;
+const prompt = 'What is the latest version of Python and what are its main features?';
+
+const schema = z.object({
+  version: z.string().describe('The latest version'),
+  release_date: z.string().describe('The release date of latest version'),
+  major_features: z.array(z.string()),
+});
+
+try {
+  const response = await searchScraper(apiKey, prompt, schema);
+  console.log(response.result);
+} catch (error) {
+  console.error(error);
+}

scrapegraph-js/examples/searchScraper_example.js

@@ -0,0 +1,12 @@
+import { searchScraper } from 'scrapegraph-js';
+import 'dotenv/config';
+
+const apiKey = process.env.SGAI_APIKEY;
+const prompt = 'What is the latest version of Python and what are its main features?';
+
+try {
+  const response = await searchScraper(apiKey, prompt);
+  console.log(response);
+} catch (error) {
+  console.error(error);
+}

scrapegraph-js/index.js

@@ -1,5 +1,6 @@
 export { smartScraper, getSmartScraperRequest } from './src/smartScraper.js';
 export { markdownify, getMarkdownifyRequest } from './src/markdownify.js';
 export { localScraper, getLocalScraperRequest } from './src/localScraper.js';
+export { searchScraper, getSearchScraperRequest } from './src/searchScraper.js';
 export { getCredits } from './src/credits.js';
 export { sendFeedback } from './src/feedback.js';

scrapegraph-js/src/searchScraper.js

@@ -0,0 +1,66 @@
+import axios from 'axios';
+import handleError from './utils/handleError.js';
+import { ZodType } from 'zod';
+import { zodToJsonSchema } from 'zod-to-json-schema';
+
+/**
+ * Search and extract information from multiple web sources using AI.
+ *
+ * @param {string} apiKey - Your ScrapeGraph AI API key
+ * @param {string} prompt - Natural language prompt describing what data to extract
+ * @param {Object} [schema] - Optional schema object defining the output structure
+ * @param {String} userAgent - the user agent like "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+ * @returns {Promise<string>} Extracted data in JSON format matching the provided schema
+ * @throws - Will throw an error in case of an HTTP failure.
+ */
+export async function searchScraper(apiKey, prompt, schema = null, userAgent = null) {
+  const endpoint = 'https://api.scrapegraphai.com/v1/searchscraper';
+  const headers = {
+    'accept': 'application/json',
+    'SGAI-APIKEY': apiKey,
+    'Content-Type': 'application/json',
+  };
+
+  if (userAgent) headers['User-Agent'] = userAgent;
+
+  const payload = {
+    user_prompt: prompt,
+  };
+
+  if (schema) {
+    if (schema instanceof ZodType) {
+      payload.output_schema = zodToJsonSchema(schema);
+    } else {
+      throw new Error('The schema must be an instance of a valid Zod schema');
+    }
+  }
+
+  try {
+    const response = await axios.post(endpoint, payload, { headers });
+    return response.data;
+  } catch (error) {
+    handleError(error);
+  }
+}
+
+/**
+ * Retrieve the status or the result of searchScraper request. It also allows you to see the result of old requests.
+ *
+ * @param {string} apiKey - Your ScrapeGraph AI API key
+ * @param {string} requestId - The request ID associated with the output of a searchScraper request.
+ * @returns {Promise<string>} Information related to the status or result of a scraping request.
+ */
+export async function getSearchScraperRequest(apiKey, requestId) {
+  const endpoint = 'https://api.scrapegraphai.com/v1/searchscraper/' + requestId;
+  const headers = {
+    'accept': 'application/json',
+    'SGAI-APIKEY': apiKey,
+  };
+
+  try {
+    const response = await axios.get(endpoint, { headers });
+    return response.data;
+  } catch (error) {
+    handleError(error);
+  }
+}