feat: add Firecrawl provider

AGENTS.md

@@ -57,6 +57,22 @@ test/unit/                # Public behavior and provider contract tests
 - Keep capability provider lists single-source: `src/core/read.ts` exports read-capable names; AI/OpenCode/Pi surfaces import that list instead of mirroring `['jina']`
 - Default to minimal dependencies; browser rendering/crawling belongs in a future read package unless explicitly decided otherwise
 
+## ADDING A NEW PROVIDER
+
+Seven files must be updated. Missing any causes a bug (test failure, missing from CLI/Pi, or silent no-op). Checklist:
+
+1. `src/providers/<name>.ts` — implement provider, call `register()` at module level; support search, read, or both
+2. `src/providers/index.ts` — add `import './<name>.ts'`
+3. `src/core/providers.ts` — add to `builtinProviders` array
+4. `src/core/resolve.ts` — add env var to `envKeys` map (unless self-hosted like searxng)
+5. `src/core/read.ts` — add to `readProviderNames` if provider supports read/scrape
+6. `packages/pi/extensions/askweb.ts` — add to `PROVIDERS` array + update tool descriptions
+7. `test/unit/<name>.ts` + `test/index.test.ts` — add provider tests + update hardcoded expected list
+
+After: `pnpm typecheck && pnpm test:run && pnpm build`
+
+Note: Pi tool descriptions (`PROVIDERS` array, description strings) are frozen at session start. After changing them, a new Pi session is required for the tools to accept the new provider name.
+
 ## ANTI-PATTERNS
 
 - Do not leak provider-specific response formats into public API

packages/pi/extensions/askweb.ts

@@ -52,7 +52,7 @@ function loadAskweb(): Promise<AskwebModule> {
   return askwebModulePromise
 }
 
-const PROVIDERS = ["auto", "all", "brave", "exa", "jina", "searxng", "serpapi", "serpbase", "tavily"] as const
+const PROVIDERS = ["auto", "all", "brave", "exa", "firecrawl", "jina", "searxng", "serpapi", "serpbase", "tavily"] as const
 const PROVIDER_HINT = `Provider to use. One of: ${PROVIDERS.join(", ")}. "auto" (or omit) picks the first available provider from env. Use "all" to query every configured provider in parallel.`
 const READ_PROVIDER_HINT = "Read provider to use. Defaults to Jina and is validated against askweb.readProviderNames at execution time."
 
@@ -132,7 +132,7 @@ export default function askwebExtension(pi: ExtensionAPI) {
     name: "askweb",
     label: "Askweb Search",
     description:
-      "Read-only/open-world network search: query one configured provider (Brave, Exa, Jina, Tavily, SerpAPI, SerpBase, SearXNG) or fan out to every available provider with provider=all. Always returns {url, title, snippet}; optional fields vary by provider: Exa adds summary/highlights/full text + score/author/image, Jina adds content/text + published date/image/metadata, Tavily adds full raw_content + score, Brave adds extra_snippets, SerpAPI adds thumbnail + position metadata, SerpBase adds Google SERP rank/request metadata, SearXNG adds engine metadata. Pick provider for the shape you need.",
+      "Read-only/open-world network search: query one configured provider (Brave, Exa, Firecrawl, Jina, Tavily, SerpAPI, SerpBase, SearXNG) or fan out to every available provider with provider=all. Always returns {url, title, snippet}; optional fields vary by provider: Exa adds summary/highlights/full text + score/author/image, Firecrawl adds markdown content from scraped pages, Jina adds content/text + published date/image/metadata, Tavily adds full raw_content + score, Brave adds extra_snippets, SerpAPI adds thumbnail + position metadata, SerpBase adds Google SERP rank/request metadata, SearXNG adds engine metadata. Pick provider for the shape you need.",
     promptSnippet:
       "Search the web with askweb. Use provider=all to query every configured provider in parallel.",
     promptGuidelines: [
@@ -234,7 +234,7 @@ export default function askwebExtension(pi: ExtensionAPI) {
     name: "askweb_read",
     label: "Askweb Read",
     description:
-      "Read-only/open-world network fetch: read a URL into normalized content using a read-capable provider. Defaults to Jina Reader (r.jina.ai). Returns URL, title/description when available, canonical content, and optional text/html/images/metadata.",
+      "Read-only/open-world network fetch: read a URL into normalized content using a read-capable provider. Defaults to Jina Reader (r.jina.ai); Firecrawl is also available for JS-rendered pages, PDFs, and structured extraction. Returns URL, title/description when available, canonical content, and optional text/html/images/metadata.",
     promptSnippet: "Read a URL with askweb_read when page content is needed, not just search results.",
     promptGuidelines: [
       "Use askweb_read after search when the user needs the contents of a specific URL.",

src/core/providers.ts

@@ -1,6 +1,7 @@
 export const builtinProviders = [
   'brave',
   'exa',
+  'firecrawl',
   'jina',
   'searxng',
   'serpapi',

src/core/read.ts

@@ -3,7 +3,7 @@ import { builtinProviders } from './providers.ts'
 import { EmptyUrlError, ReadNotSupportedError } from './errors.ts'
 import { create } from './registry.ts'
 
-export const readProviderNames = ['jina'] as const
+export const readProviderNames = ['jina', 'firecrawl'] as const
 export type ReadProviderName = typeof readProviderNames[number]
 
 export interface ReadUrlOptions extends ReadOptions {

src/core/resolve.ts

@@ -5,6 +5,7 @@ import { NoProviderAvailableError, NoProviderConfiguredError } from './errors.ts
 const envKeys: Record<string, WebSearchProviderName> = {
   EXA_API_KEY: 'exa',
   BRAVE_API_KEY: 'brave',
+  FIRECRAWL_API_KEY: 'firecrawl',
   JINA_API_KEY: 'jina',
   TAVILY_API_KEY: 'tavily',
   SERPAPI_API_KEY: 'serpapi',

src/providers/firecrawl.ts

@@ -0,0 +1,168 @@
+import type { SearchResult, SearchOptions, ReadResult, ReadOptions, SearchProvider, ProviderConfig, ProviderFactory } from '../core/types.ts'
+import { defaultClient } from '../core/client.ts'
+import type { Client } from '../core/client.ts'
+import { AuthError, normalizeError } from '../core/errors.ts'
+import { register } from '../core/registry.ts'
+
+interface FirecrawlWebResult {
+  title: string
+  description: string
+  url: string
+  markdown?: string
+  html?: string
+  links?: string[]
+  position?: number
+  metadata?: {
+    title?: string
+    description?: string
+    sourceURL?: string
+    statusCode?: number
+    error?: string
+  }
+}
+
+interface FirecrawlSearchResponse {
+  success: boolean
+  data?: {
+    web?: FirecrawlWebResult[]
+    news?: FirecrawlWebResult[]
+    warning?: string
+  }
+}
+
+interface FirecrawlScrapeResponse {
+  success: boolean
+  data?: {
+    markdown?: string
+    html?: string
+    metadata?: {
+      title?: string
+      description?: string
+      sourceURL?: string
+      language?: string
+      keywords?: string
+      ogImage?: string
+      [key: string]: unknown
+    }
+    links?: string[]
+    warning?: string
+  }
+}
+
+const FIRECRAWL_MAX_RESULTS = 100
+
+function clampMaxResults(max?: number): number {
+  return Math.min(Math.max(max ?? 10, 1), FIRECRAWL_MAX_RESULTS)
+}
+
+class FirecrawlProvider implements SearchProvider {
+  private readonly client: Client
+  private readonly baseURL: string
+  private readonly apiKey: string
+
+  constructor(config: ProviderConfig) {
+    if (!config.apiKey) {
+      throw new AuthError('Missing API key for Firecrawl. Set FIRECRAWL_API_KEY', 'firecrawl')
+    }
+
+    this.client = defaultClient()
+    this.baseURL = (config.baseURL ?? 'https://api.firecrawl.dev').replace(/\/+$/, '')
+    this.apiKey = config.apiKey
+  }
+
+  name(): string {
+    return 'firecrawl'
+  }
+
+  private authHeaders(): Record<string, string> {
+    return { 'Authorization': `Bearer ${this.apiKey}` }
+  }
+
+  async search(query: string, options?: SearchOptions): Promise<SearchResult[]> {
+    const body: Record<string, unknown> = {
+      query,
+      limit: clampMaxResults(options?.maxResults),
+    }
+
+    if (options?.includeDomains?.length) {
+      body.includeDomains = options.includeDomains
+    }
+    if (options?.excludeDomains?.length) {
+      body.excludeDomains = options.excludeDomains
+    }
+    if (options?.category === 'news') {
+      body.sources = ['news']
+    }
+
+    try {
+      const url = `${this.baseURL}/v2/search`
+      const response = await this.client.postJSON<FirecrawlSearchResponse>(url, body, this.authHeaders())
+
+      if (!response.success) {
+        throw new Error('Firecrawl search failed')
+      }
+
+      const web = response.data?.web ?? []
+      const news = response.data?.news ?? []
+      const allResults = news.length > 0 ? [...web, ...news] : web
+      return allResults.slice(0, clampMaxResults(options?.maxResults)).map(mapSearchResult)
+    }
+    catch (error) {
+      throw normalizeError(error, 'firecrawl')
+    }
+  }
+
+  async read(url: string, options?: ReadOptions): Promise<ReadResult> {
+    const body: Record<string, unknown> = {
+      url,
+      formats: [normalizeFormat(options?.format)],
+      onlyMainContent: true,
+    }
+
+    if (options?.timeout) {
+      body.timeout = options.timeout * 1000
+    }
+
+    try {
+      const endpoint = `${this.baseURL}/v2/scrape`
+      const response = await this.client.postJSON<FirecrawlScrapeResponse>(endpoint, body, this.authHeaders())
+
+      if (!response.success) {
+        throw new Error('Firecrawl scrape failed')
+      }
+
+      const data = response.data ?? {}
+      return {
+        url,
+        title: data.metadata?.title,
+        description: data.metadata?.description,
+        content: data.markdown ?? data.html ?? '',
+        html: data.html,
+        links: data.links,
+        image: data.metadata?.ogImage,
+        metadata: data.metadata,
+      }
+    }
+    catch (error) {
+      throw normalizeError(error, 'firecrawl')
+    }
+  }
+}
+
+function normalizeFormat(format?: string): 'markdown' | 'html' {
+  if (format === 'html') return 'html'
+  return 'markdown'
+}
+
+function mapSearchResult(result: FirecrawlWebResult): SearchResult {
+  return {
+    url: result.url,
+    title: result.title,
+    snippet: result.description,
+    text: result.markdown,
+  }
+}
+
+const factory: ProviderFactory = (config) => new FirecrawlProvider(config)
+
+register('firecrawl', 'https://api.firecrawl.dev', factory)

src/providers/index.ts

@@ -5,3 +5,4 @@ import './tavily.ts'
 import './serpapi.ts'
 import './serpbase.ts'
 import './searxng.ts'
+import './firecrawl.ts'

test/index.test.ts

@@ -7,7 +7,7 @@ describe('askweb', () => {
   })
 
   it('should list all built-in provider names', () => {
-    expect(builtinProviders).toEqual(['brave', 'exa', 'jina', 'searxng', 'serpapi', 'serpbase', 'tavily'])
+    expect(builtinProviders).toEqual(['brave', 'exa', 'firecrawl', 'jina', 'searxng', 'serpapi', 'serpbase', 'tavily'])
   })
 
   it('should register built-in providers from main entrypoint', () => {

test/unit/ai-tool.test.ts

@@ -62,7 +62,7 @@ const searxngResponse = {
 }
 
 const savedEnv: Record<string, string | undefined> = {}
-const envKeys = ['EXA_API_KEY', 'BRAVE_API_KEY', 'JINA_API_KEY', 'TAVILY_API_KEY', 'SERPAPI_API_KEY', 'SERPBASE_API_KEY']
+const envKeys = ['EXA_API_KEY', 'BRAVE_API_KEY', 'FIRECRAWL_API_KEY', 'JINA_API_KEY', 'TAVILY_API_KEY', 'SERPAPI_API_KEY', 'SERPBASE_API_KEY']
 
 describe('searchTool', () => {
   beforeEach(() => {