From cabb269251ad8aa65d39c6f3004cd9395e3aaed8 Mon Sep 17 00:00:00 2001 From: oritwoen <18102267+oritwoen@users.noreply.github.com> Date: Sun, 24 May 2026 23:03:41 +0200 Subject: [PATCH 1/3] feat: add Firecrawl provider for search and read --- AGENTS.md | 16 ++ packages/pi/extensions/askweb.ts | 6 +- src/core/providers.ts | 1 + src/core/read.ts | 2 +- src/core/resolve.ts | 1 + src/providers/firecrawl.ts | 159 ++++++++++++++++++ src/providers/index.ts | 1 + test/index.test.ts | 2 +- test/unit/firecrawl.test.ts | 271 +++++++++++++++++++++++++++++++ 9 files changed, 454 insertions(+), 5 deletions(-) create mode 100644 src/providers/firecrawl.ts create mode 100644 test/unit/firecrawl.test.ts diff --git a/AGENTS.md b/AGENTS.md index 0e23e8b..cee1ffc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -57,6 +57,22 @@ test/unit/ # Public behavior and provider contract tests - Keep capability provider lists single-source: `src/core/read.ts` exports read-capable names; AI/OpenCode/Pi surfaces import that list instead of mirroring `['jina']` - Default to minimal dependencies; browser rendering/crawling belongs in a future read package unless explicitly decided otherwise +## ADDING A NEW PROVIDER + +Seven files must be updated. Missing any causes a bug (test failure, missing from CLI/Pi, or silent no-op). Checklist: + +1. `src/providers/.ts` — implement `SearchProvider`, call `register()` at module level +2. `src/providers/index.ts` — add `import './.ts'` +3. `src/core/providers.ts` — add to `builtinProviders` array +4. `src/core/resolve.ts` — add env var to `envKeys` map (unless self-hosted like searxng) +5. `src/core/read.ts` — add to `readProviderNames` if provider supports read/scrape +6. `packages/pi/extensions/askweb.ts` — add to `PROVIDERS` array + update tool descriptions +7. `test/unit/.ts` + `test/index.test.ts` — add provider tests + update hardcoded expected list + +After: `pnpm typecheck && pnpm test:run && pnpm build` + +Note: Pi tool descriptions (`PROVIDERS` array, description strings) are frozen at session start. After changing them, a new Pi session is required for the tools to accept the new provider name. + ## ANTI-PATTERNS - Do not leak provider-specific response formats into public API diff --git a/packages/pi/extensions/askweb.ts b/packages/pi/extensions/askweb.ts index 731f39c..11658a3 100644 --- a/packages/pi/extensions/askweb.ts +++ b/packages/pi/extensions/askweb.ts @@ -52,7 +52,7 @@ function loadAskweb(): Promise { return askwebModulePromise } -const PROVIDERS = ["auto", "all", "brave", "exa", "jina", "searxng", "serpapi", "serpbase", "tavily"] as const +const PROVIDERS = ["auto", "all", "brave", "exa", "firecrawl", "jina", "searxng", "serpapi", "serpbase", "tavily"] as const const PROVIDER_HINT = `Provider to use. One of: ${PROVIDERS.join(", ")}. "auto" (or omit) picks the first available provider from env. Use "all" to query every configured provider in parallel.` const READ_PROVIDER_HINT = "Read provider to use. Defaults to Jina and is validated against askweb.readProviderNames at execution time." @@ -132,7 +132,7 @@ export default function askwebExtension(pi: ExtensionAPI) { name: "askweb", label: "Askweb Search", description: - "Read-only/open-world network search: query one configured provider (Brave, Exa, Jina, Tavily, SerpAPI, SerpBase, SearXNG) or fan out to every available provider with provider=all. Always returns {url, title, snippet}; optional fields vary by provider: Exa adds summary/highlights/full text + score/author/image, Jina adds content/text + published date/image/metadata, Tavily adds full raw_content + score, Brave adds extra_snippets, SerpAPI adds thumbnail + position metadata, SerpBase adds Google SERP rank/request metadata, SearXNG adds engine metadata. Pick provider for the shape you need.", + "Read-only/open-world network search: query one configured provider (Brave, Exa, Firecrawl, Jina, Tavily, SerpAPI, SerpBase, SearXNG) or fan out to every available provider with provider=all. Always returns {url, title, snippet}; optional fields vary by provider: Exa adds summary/highlights/full text + score/author/image, Firecrawl adds markdown content from scraped pages, Jina adds content/text + published date/image/metadata, Tavily adds full raw_content + score, Brave adds extra_snippets, SerpAPI adds thumbnail + position metadata, SerpBase adds Google SERP rank/request metadata, SearXNG adds engine metadata. Pick provider for the shape you need.", promptSnippet: "Search the web with askweb. Use provider=all to query every configured provider in parallel.", promptGuidelines: [ @@ -234,7 +234,7 @@ export default function askwebExtension(pi: ExtensionAPI) { name: "askweb_read", label: "Askweb Read", description: - "Read-only/open-world network fetch: read a URL into normalized content using a read-capable provider. Defaults to Jina Reader (r.jina.ai). Returns URL, title/description when available, canonical content, and optional text/html/images/metadata.", + "Read-only/open-world network fetch: read a URL into normalized content using a read-capable provider. Defaults to Jina Reader (r.jina.ai); Firecrawl is also available for JS-rendered pages, PDFs, and structured extraction. Returns URL, title/description when available, canonical content, and optional text/html/images/metadata.", promptSnippet: "Read a URL with askweb_read when page content is needed, not just search results.", promptGuidelines: [ "Use askweb_read after search when the user needs the contents of a specific URL.", diff --git a/src/core/providers.ts b/src/core/providers.ts index 4c3d56b..3675e01 100644 --- a/src/core/providers.ts +++ b/src/core/providers.ts @@ -1,6 +1,7 @@ export const builtinProviders = [ 'brave', 'exa', + 'firecrawl', 'jina', 'searxng', 'serpapi', diff --git a/src/core/read.ts b/src/core/read.ts index 4094daa..3e63195 100644 --- a/src/core/read.ts +++ b/src/core/read.ts @@ -3,7 +3,7 @@ import { builtinProviders } from './providers.ts' import { EmptyUrlError, ReadNotSupportedError } from './errors.ts' import { create } from './registry.ts' -export const readProviderNames = ['jina'] as const +export const readProviderNames = ['jina', 'firecrawl'] as const export type ReadProviderName = typeof readProviderNames[number] export interface ReadUrlOptions extends ReadOptions { diff --git a/src/core/resolve.ts b/src/core/resolve.ts index fd5be61..ff69574 100644 --- a/src/core/resolve.ts +++ b/src/core/resolve.ts @@ -5,6 +5,7 @@ import { NoProviderAvailableError, NoProviderConfiguredError } from './errors.ts const envKeys: Record = { EXA_API_KEY: 'exa', BRAVE_API_KEY: 'brave', + FIRECRAWL_API_KEY: 'firecrawl', JINA_API_KEY: 'jina', TAVILY_API_KEY: 'tavily', SERPAPI_API_KEY: 'serpapi', diff --git a/src/providers/firecrawl.ts b/src/providers/firecrawl.ts new file mode 100644 index 0000000..0f091dd --- /dev/null +++ b/src/providers/firecrawl.ts @@ -0,0 +1,159 @@ +import type { SearchResult, SearchOptions, ReadResult, ReadOptions, SearchProvider, ProviderConfig, ProviderFactory } from '../core/types.ts' +import { defaultClient } from '../core/client.ts' +import type { Client } from '../core/client.ts' +import { AuthError, normalizeError } from '../core/errors.ts' +import { register } from '../core/registry.ts' + +interface FirecrawlWebResult { + title: string + description: string + url: string + markdown?: string + html?: string + links?: string[] + position?: number + metadata?: { + title?: string + description?: string + sourceURL?: string + statusCode?: number + error?: string + } +} + +interface FirecrawlSearchResponse { + success: boolean + data?: { + web?: FirecrawlWebResult[] + warning?: string + } +} + +interface FirecrawlScrapeResponse { + success: boolean + data?: { + markdown?: string + html?: string + metadata?: { + title?: string + description?: string + sourceURL?: string + language?: string + keywords?: string + ogImage?: string + [key: string]: unknown + } + links?: string[] + warning?: string + } +} + +const FIRECRAWL_MAX_RESULTS = 100 + +function clampMaxResults(max?: number): number { + return Math.min(Math.max(max ?? 10, 1), FIRECRAWL_MAX_RESULTS) +} + +class FirecrawlProvider implements SearchProvider { + private readonly client: Client + private readonly baseURL: string + private readonly apiKey: string + + constructor(config: ProviderConfig) { + if (!config.apiKey) { + throw new AuthError('Missing API key for Firecrawl. Set FIRECRAWL_API_KEY', 'firecrawl') + } + + this.client = defaultClient() + this.baseURL = (config.baseURL ?? 'https://api.firecrawl.dev').replace(/\/+$/, '') + this.apiKey = config.apiKey + } + + name(): string { + return 'firecrawl' + } + + private authHeaders(): Record { + return { 'Authorization': `Bearer ${this.apiKey}` } + } + + async search(query: string, options?: SearchOptions): Promise { + const body: Record = { + query, + limit: clampMaxResults(options?.maxResults), + } + + if (options?.includeDomains?.length) { + body.includeDomains = options.includeDomains + } + if (options?.excludeDomains?.length) { + body.excludeDomains = options.excludeDomains + } + if (options?.category === 'news') { + body.sources = ['news'] + } + + try { + const url = `${this.baseURL}/v2/search` + const response = await this.client.postJSON(url, body, this.authHeaders()) + + if (!response.success) { + throw new Error('Firecrawl search failed') + } + + return (response.data?.web ?? []).map(mapSearchResult) + } + catch (error) { + throw normalizeError(error, 'firecrawl') + } + } + + async read(url: string, options?: ReadOptions): Promise { + const body: Record = { + url, + formats: [options?.format ?? 'markdown'], + onlyMainContent: true, + } + + if (options?.timeout) { + body.timeout = options.timeout * 1000 + } + + try { + const endpoint = `${this.baseURL}/v2/scrape` + const response = await this.client.postJSON(endpoint, body, this.authHeaders()) + + if (!response.success) { + throw new Error('Firecrawl scrape failed') + } + + const data = response.data ?? {} + return { + url, + title: data.metadata?.title, + description: data.metadata?.description, + content: data.markdown ?? '', + html: data.html, + links: data.links, + image: data.metadata?.ogImage, + metadata: data.metadata, + } + } + catch (error) { + throw normalizeError(error, 'firecrawl') + } + } +} + +function mapSearchResult(result: FirecrawlWebResult): SearchResult { + return { + url: result.url, + title: result.title, + snippet: result.description, + text: result.markdown, + } +} + +const factory: ProviderFactory = (config) => new FirecrawlProvider(config) + +register('firecrawl', 'https://api.firecrawl.dev', factory) diff --git a/src/providers/index.ts b/src/providers/index.ts index d108bee..f7effd3 100644 --- a/src/providers/index.ts +++ b/src/providers/index.ts @@ -5,3 +5,4 @@ import './tavily.ts' import './serpapi.ts' import './serpbase.ts' import './searxng.ts' +import './firecrawl.ts' diff --git a/test/index.test.ts b/test/index.test.ts index c647c36..39bb8d2 100644 --- a/test/index.test.ts +++ b/test/index.test.ts @@ -7,7 +7,7 @@ describe('askweb', () => { }) it('should list all built-in provider names', () => { - expect(builtinProviders).toEqual(['brave', 'exa', 'jina', 'searxng', 'serpapi', 'serpbase', 'tavily']) + expect(builtinProviders).toEqual(['brave', 'exa', 'firecrawl', 'jina', 'searxng', 'serpapi', 'serpbase', 'tavily']) }) it('should register built-in providers from main entrypoint', () => { diff --git a/test/unit/firecrawl.test.ts b/test/unit/firecrawl.test.ts new file mode 100644 index 0000000..7f4dc4a --- /dev/null +++ b/test/unit/firecrawl.test.ts @@ -0,0 +1,271 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' + +const mockPostJSON = vi.fn() + +vi.mock('../../src/core/client.ts', () => ({ + Client: vi.fn(), + defaultClient: vi.fn(() => ({ + postJSON: mockPostJSON, + getJSON: vi.fn(), + maxRetries: 5, + baseDelay: 50, + timeout: 30000, + userAgent: 'askweb/0.0.1', + })), +})) + +import { create, has } from '../../src/core/registry.ts' +import { AuthError } from '../../src/core/errors.ts' +import type { SearchResult } from '../../src/core/types.ts' + +// Triggers self-registration of firecrawl provider +import '../../src/providers/index.ts' + +const firecrawlSearchResponse = { + success: true, + data: { + web: [ + { + title: 'Firecrawl - Web Scraping API', + description: 'Turn websites into LLM-ready data.', + url: 'https://www.firecrawl.dev/', + position: 1, + }, + { + title: 'GitHub - firecrawl/firecrawl', + description: 'Open source web scraping API.', + url: 'https://github.com/firecrawl/firecrawl', + position: 2, + markdown: '# Firecrawl\n\nOpen source web scraper.', + }, + ], + }, +} + +const firecrawlScrapeResponse = { + success: true, + data: { + markdown: '# Firecrawl\n\nThe web scraping API for AI.', + html: '

Firecrawl

', + metadata: { + title: 'Firecrawl', + description: 'The web scraping API for AI.', + sourceURL: 'https://www.firecrawl.dev/', + language: 'en', + ogImage: 'https://www.firecrawl.dev/og.png', + }, + links: ['https://www.firecrawl.dev/pricing', 'https://docs.firecrawl.dev'], + }, +} + +describe('firecrawl provider', () => { + beforeEach(() => { + mockPostJSON.mockReset() + mockPostJSON.mockResolvedValue(firecrawlSearchResponse) + delete process.env.FIRECRAWL_API_KEY + }) + + describe('self-registration', () => { + it('registers itself on import', () => { + expect(has('firecrawl')).toBe(true) + }) + }) + + describe('create', () => { + it('creates provider with apiKey', () => { + expect(() => create('firecrawl', { apiKey: 'test-key' })).not.toThrow() + }) + + it('throws AuthError without apiKey and without env var', () => { + expect(() => create('firecrawl', {})).toThrow(AuthError) + }) + }) + + describe('name()', () => { + it('returns firecrawl', () => { + const provider = create('firecrawl', { apiKey: 'test-key' }) + expect(provider.name()).toBe('firecrawl') + }) + }) + + describe('search()', () => { + it('calls postJSON with correct url and Authorization header', async () => { + const provider = create('firecrawl', { apiKey: 'fc-test-key' }) + await provider.search('test query') + + expect(mockPostJSON).toHaveBeenCalledOnce() + const [url, body, headers] = mockPostJSON.mock.calls[0] + + expect(url).toBe('https://api.firecrawl.dev/v2/search') + expect(body).toMatchObject({ + query: 'test query', + limit: 10, + }) + expect(headers).toMatchObject({ + 'Authorization': 'Bearer fc-test-key', + }) + }) + + it('maps result fields correctly', async () => { + const provider = create('firecrawl', { apiKey: 'test-key' }) + const results: SearchResult[] = await provider.search('test query') + + expect(results).toHaveLength(2) + expect(results[0].url).toBe('https://www.firecrawl.dev/') + expect(results[0].title).toBe('Firecrawl - Web Scraping API') + expect(results[0].snippet).toBe('Turn websites into LLM-ready data.') + }) + + it('maps markdown content to text field', async () => { + const provider = create('firecrawl', { apiKey: 'test-key' }) + const results: SearchResult[] = await provider.search('test query') + + expect(results[1].text).toBe('# Firecrawl\n\nOpen source web scraper.') + expect(results[0].text).toBeUndefined() + }) + + it('maps maxResults to limit in body', async () => { + const provider = create('firecrawl', { apiKey: 'test-key' }) + await provider.search('test query', { maxResults: 5 }) + + const [, body] = mockPostJSON.mock.calls[0] + expect(body.limit).toBe(5) + }) + + it('passes includeDomains in body', async () => { + const provider = create('firecrawl', { apiKey: 'test-key' }) + await provider.search('test query', { includeDomains: ['github.com'] }) + + const [, body] = mockPostJSON.mock.calls[0] + expect(body.includeDomains).toEqual(['github.com']) + }) + + it('passes excludeDomains in body', async () => { + const provider = create('firecrawl', { apiKey: 'test-key' }) + await provider.search('test query', { excludeDomains: ['reddit.com'] }) + + const [, body] = mockPostJSON.mock.calls[0] + expect(body.excludeDomains).toEqual(['reddit.com']) + }) + + it('sets sources to news when category is news', async () => { + const provider = create('firecrawl', { apiKey: 'test-key' }) + await provider.search('test query', { category: 'news' }) + + const [, body] = mockPostJSON.mock.calls[0] + expect(body.sources).toEqual(['news']) + }) + + it('does not set sources when category is not news', async () => { + const provider = create('firecrawl', { apiKey: 'test-key' }) + await provider.search('test query', { category: 'general' }) + + const [, body] = mockPostJSON.mock.calls[0] + expect(body.sources).toBeUndefined() + }) + + it('returns empty array when web results are missing', async () => { + mockPostJSON.mockResolvedValueOnce({ success: true, data: {} }) + + const provider = create('firecrawl', { apiKey: 'test-key' }) + const results = await provider.search('query') + + expect(results).toEqual([]) + }) + + it('throws when success is false', async () => { + mockPostJSON.mockResolvedValueOnce({ success: false }) + + const provider = create('firecrawl', { apiKey: 'test-key' }) + await expect(provider.search('query')).rejects.toThrow() + }) + + it('clamps maxResults to 100', async () => { + const provider = create('firecrawl', { apiKey: 'test-key' }) + await provider.search('test query', { maxResults: 500 }) + + const [, body] = mockPostJSON.mock.calls[0] + expect(body.limit).toBe(100) + }) + }) + + describe('read()', () => { + beforeEach(() => { + mockPostJSON.mockReset() + mockPostJSON.mockResolvedValue(firecrawlScrapeResponse) + }) + + it('calls postJSON with scrape endpoint and url in body', async () => { + const provider = create('firecrawl', { apiKey: 'fc-test-key' }) + await provider.read('https://example.com') + + expect(mockPostJSON).toHaveBeenCalledOnce() + const [url, body, headers] = mockPostJSON.mock.calls[0] + + expect(url).toBe('https://api.firecrawl.dev/v2/scrape') + expect(body).toMatchObject({ + url: 'https://example.com', + formats: ['markdown'], + onlyMainContent: true, + }) + expect(headers).toMatchObject({ + 'Authorization': 'Bearer fc-test-key', + }) + }) + + it('returns read result with content from markdown', async () => { + const provider = create('firecrawl', { apiKey: 'test-key' }) + const result = await provider.read('https://example.com') + + expect(result.url).toBe('https://example.com') + expect(result.title).toBe('Firecrawl') + expect(result.description).toBe('The web scraping API for AI.') + expect(result.content).toBe('# Firecrawl\n\nThe web scraping API for AI.') + expect(result.html).toBe('

Firecrawl

') + expect(result.links).toEqual(['https://www.firecrawl.dev/pricing', 'https://docs.firecrawl.dev']) + expect(result.image).toBe('https://www.firecrawl.dev/og.png') + }) + + it('passes format option to formats array', async () => { + const provider = create('firecrawl', { apiKey: 'test-key' }) + await provider.read('https://example.com', { format: 'html' }) + + const [, body] = mockPostJSON.mock.calls[0] + expect(body.formats).toEqual(['html']) + }) + + it('converts timeout from seconds to milliseconds', async () => { + const provider = create('firecrawl', { apiKey: 'test-key' }) + await provider.read('https://example.com', { timeout: 30 }) + + const [, body] = mockPostJSON.mock.calls[0] + expect(body.timeout).toBe(30000) + }) + + it('sets onlyMainContent to true by default', async () => { + const provider = create('firecrawl', { apiKey: 'test-key' }) + await provider.read('https://example.com') + + const [, body] = mockPostJSON.mock.calls[0] + expect(body.onlyMainContent).toBe(true) + }) + + it('throws when success is false', async () => { + mockPostJSON.mockResolvedValueOnce({ success: false }) + + const provider = create('firecrawl', { apiKey: 'test-key' }) + await expect(provider.read('https://example.com')).rejects.toThrow() + }) + + it('handles missing data gracefully', async () => { + mockPostJSON.mockResolvedValueOnce({ success: true, data: {} }) + + const provider = create('firecrawl', { apiKey: 'test-key' }) + const result = await provider.read('https://example.com') + + expect(result.content).toBe('') + expect(result.title).toBeUndefined() + expect(result.links).toBeUndefined() + }) + }) +}) From 3bd90817e14e8fe3db0d87412be5b06aa7a92a6d Mon Sep 17 00:00:00 2001 From: oritwoen <18102267+oritwoen@users.noreply.github.com> Date: Sun, 24 May 2026 23:48:21 +0200 Subject: [PATCH 2/3] fix: address review feedback for Firecrawl provider --- AGENTS.md | 2 +- src/providers/firecrawl.ts | 14 +++++++++--- test/unit/ai-tool.test.ts | 2 +- test/unit/firecrawl.test.ts | 40 +++++++++++++++++++++++++++++++++ test/unit/resolve-async.test.ts | 2 +- test/unit/resolve.test.ts | 2 +- 6 files changed, 55 insertions(+), 7 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index cee1ffc..bf0f110 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -61,7 +61,7 @@ test/unit/ # Public behavior and provider contract tests Seven files must be updated. Missing any causes a bug (test failure, missing from CLI/Pi, or silent no-op). Checklist: -1. `src/providers/.ts` — implement `SearchProvider`, call `register()` at module level +1. `src/providers/.ts` — implement provider, call `register()` at module level; support search, read, or both 2. `src/providers/index.ts` — add `import './.ts'` 3. `src/core/providers.ts` — add to `builtinProviders` array 4. `src/core/resolve.ts` — add env var to `envKeys` map (unless self-hosted like searxng) diff --git a/src/providers/firecrawl.ts b/src/providers/firecrawl.ts index 0f091dd..a90a453 100644 --- a/src/providers/firecrawl.ts +++ b/src/providers/firecrawl.ts @@ -25,6 +25,7 @@ interface FirecrawlSearchResponse { success: boolean data?: { web?: FirecrawlWebResult[] + news?: FirecrawlWebResult[] warning?: string } } @@ -101,7 +102,9 @@ class FirecrawlProvider implements SearchProvider { throw new Error('Firecrawl search failed') } - return (response.data?.web ?? []).map(mapSearchResult) + const web = response.data?.web ?? [] + const news = response.data?.news ?? [] + return (news.length > 0 ? [...web, ...news] : web).map(mapSearchResult) } catch (error) { throw normalizeError(error, 'firecrawl') @@ -111,7 +114,7 @@ class FirecrawlProvider implements SearchProvider { async read(url: string, options?: ReadOptions): Promise { const body: Record = { url, - formats: [options?.format ?? 'markdown'], + formats: [normalizeFormat(options?.format)], onlyMainContent: true, } @@ -132,7 +135,7 @@ class FirecrawlProvider implements SearchProvider { url, title: data.metadata?.title, description: data.metadata?.description, - content: data.markdown ?? '', + content: data.markdown ?? data.html ?? '', html: data.html, links: data.links, image: data.metadata?.ogImage, @@ -145,6 +148,11 @@ class FirecrawlProvider implements SearchProvider { } } +function normalizeFormat(format?: string): 'markdown' | 'html' { + if (format === 'html') return 'html' + return 'markdown' +} + function mapSearchResult(result: FirecrawlWebResult): SearchResult { return { url: result.url, diff --git a/test/unit/ai-tool.test.ts b/test/unit/ai-tool.test.ts index 7facd6e..62d09b8 100644 --- a/test/unit/ai-tool.test.ts +++ b/test/unit/ai-tool.test.ts @@ -62,7 +62,7 @@ const searxngResponse = { } const savedEnv: Record = {} -const envKeys = ['EXA_API_KEY', 'BRAVE_API_KEY', 'JINA_API_KEY', 'TAVILY_API_KEY', 'SERPAPI_API_KEY', 'SERPBASE_API_KEY'] +const envKeys = ['EXA_API_KEY', 'BRAVE_API_KEY', 'FIRECRAWL_API_KEY', 'JINA_API_KEY', 'TAVILY_API_KEY', 'SERPAPI_API_KEY', 'SERPBASE_API_KEY'] describe('searchTool', () => { beforeEach(() => { diff --git a/test/unit/firecrawl.test.ts b/test/unit/firecrawl.test.ts index 7f4dc4a..38b7f97 100644 --- a/test/unit/firecrawl.test.ts +++ b/test/unit/firecrawl.test.ts @@ -156,6 +156,23 @@ describe('firecrawl provider', () => { expect(body.sources).toEqual(['news']) }) + it('combines web and news results when news data is present', async () => { + mockPostJSON.mockResolvedValueOnce({ + success: true, + data: { + web: [{ title: 'Web Result', description: 'web desc', url: 'https://example.com/web' }], + news: [{ title: 'News Result', description: 'news desc', url: 'https://example.com/news' }], + }, + }) + + const provider = create('firecrawl', { apiKey: 'test-key' }) + const results = await provider.search('test query', { category: 'news' }) + + expect(results).toHaveLength(2) + expect(results[0].title).toBe('Web Result') + expect(results[1].title).toBe('News Result') + }) + it('does not set sources when category is not news', async () => { const provider = create('firecrawl', { apiKey: 'test-key' }) await provider.search('test query', { category: 'general' }) @@ -234,6 +251,29 @@ describe('firecrawl provider', () => { expect(body.formats).toEqual(['html']) }) + it('maps text format to markdown', async () => { + const provider = create('firecrawl', { apiKey: 'test-key' }) + await provider.read('https://example.com', { format: 'text' }) + + const [, body] = mockPostJSON.mock.calls[0] + expect(body.formats).toEqual(['markdown']) + }) + + it('falls back to html content when markdown is missing', async () => { + mockPostJSON.mockResolvedValueOnce({ + success: true, + data: { + html: '

Only HTML

', + metadata: { title: 'Test' }, + }, + }) + + const provider = create('firecrawl', { apiKey: 'test-key' }) + const result = await provider.read('https://example.com', { format: 'html' }) + + expect(result.content).toBe('

Only HTML

') + }) + it('converts timeout from seconds to milliseconds', async () => { const provider = create('firecrawl', { apiKey: 'test-key' }) await provider.read('https://example.com', { timeout: 30 }) diff --git a/test/unit/resolve-async.test.ts b/test/unit/resolve-async.test.ts index 624194a..8f4f694 100644 --- a/test/unit/resolve-async.test.ts +++ b/test/unit/resolve-async.test.ts @@ -8,7 +8,7 @@ import { searchAllDetailed } from '../../src/core/all.ts' import { NoProviderAvailableError } from '../../src/core/errors.ts' import '../../src/providers/index.ts' -const envKeys = ['EXA_API_KEY', 'BRAVE_API_KEY', 'JINA_API_KEY', 'TAVILY_API_KEY', 'SERPAPI_API_KEY', 'SERPBASE_API_KEY'] as const +const envKeys = ['EXA_API_KEY', 'BRAVE_API_KEY', 'FIRECRAWL_API_KEY', 'JINA_API_KEY', 'TAVILY_API_KEY', 'SERPAPI_API_KEY', 'SERPBASE_API_KEY'] as const describe('resolve async', () => { const savedEnv: Record = {} diff --git a/test/unit/resolve.test.ts b/test/unit/resolve.test.ts index 89399ea..2b25a59 100644 --- a/test/unit/resolve.test.ts +++ b/test/unit/resolve.test.ts @@ -2,7 +2,7 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { detectAvailableProviders, resolveDefaultProvider, listProviders } from '../../src/core/resolve.ts' import '../../src/providers/index.ts' -const envKeys = ['EXA_API_KEY', 'BRAVE_API_KEY', 'JINA_API_KEY', 'TAVILY_API_KEY', 'SERPAPI_API_KEY', 'SERPBASE_API_KEY'] as const +const envKeys = ['EXA_API_KEY', 'BRAVE_API_KEY', 'FIRECRAWL_API_KEY', 'JINA_API_KEY', 'TAVILY_API_KEY', 'SERPAPI_API_KEY', 'SERPBASE_API_KEY'] as const describe('resolve', () => { const savedEnv: Record = {} From 7164326940b64b27a5f40450520586831b177ef0 Mon Sep 17 00:00:00 2001 From: oritwoen <18102267+oritwoen@users.noreply.github.com> Date: Mon, 25 May 2026 00:09:22 +0200 Subject: [PATCH 3/3] fix: slice combined web+news results to maxResults --- src/providers/firecrawl.ts | 3 ++- test/unit/firecrawl.test.ts | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/providers/firecrawl.ts b/src/providers/firecrawl.ts index a90a453..76b862e 100644 --- a/src/providers/firecrawl.ts +++ b/src/providers/firecrawl.ts @@ -104,7 +104,8 @@ class FirecrawlProvider implements SearchProvider { const web = response.data?.web ?? [] const news = response.data?.news ?? [] - return (news.length > 0 ? [...web, ...news] : web).map(mapSearchResult) + const allResults = news.length > 0 ? [...web, ...news] : web + return allResults.slice(0, clampMaxResults(options?.maxResults)).map(mapSearchResult) } catch (error) { throw normalizeError(error, 'firecrawl') diff --git a/test/unit/firecrawl.test.ts b/test/unit/firecrawl.test.ts index 38b7f97..ea02dae 100644 --- a/test/unit/firecrawl.test.ts +++ b/test/unit/firecrawl.test.ts @@ -173,6 +173,21 @@ describe('firecrawl provider', () => { expect(results[1].title).toBe('News Result') }) + it('slices combined results to maxResults', async () => { + mockPostJSON.mockResolvedValueOnce({ + success: true, + data: { + web: Array.from({ length: 5 }, (_, i) => ({ title: `Web ${i}`, description: '', url: `https://example.com/w${i}` })), + news: Array.from({ length: 5 }, (_, i) => ({ title: `News ${i}`, description: '', url: `https://example.com/n${i}` })), + }, + }) + + const provider = create('firecrawl', { apiKey: 'test-key' }) + const results = await provider.search('test query', { category: 'news', maxResults: 5 }) + + expect(results).toHaveLength(5) + }) + it('does not set sources when category is not news', async () => { const provider = create('firecrawl', { apiKey: 'test-key' }) await provider.search('test query', { category: 'general' })