From 16953c1b8f6858b2f9f3985310121b1afbc5d4be Mon Sep 17 00:00:00 2001 From: oritwoen <18102267+oritwoen@users.noreply.github.com> Date: Wed, 20 May 2026 12:34:41 +0200 Subject: [PATCH 1/6] feat: add jina search and read support --- README.md | 98 ++++++++--- package.json | 2 +- packages/pi/extensions/askweb.ts | 134 ++++++++++++++- src/ai.ts | 27 ++- src/cli-args.ts | 1 + src/cli.ts | 3 +- src/commands/read.ts | 146 +++++++++++++++++ src/core/errors.ts | 19 +++ src/core/providers.ts | 1 + src/core/read.ts | 25 +++ src/core/resolve.ts | 1 + src/core/types.ts | 25 +++ src/index.ts | 7 +- src/opencode.ts | 17 +- src/providers/index.ts | 1 + src/providers/jina.ts | 184 +++++++++++++++++++++ test/index.test.ts | 10 +- test/unit/ai-tool.test.ts | 45 ++++- test/unit/all.test.ts | 2 + test/unit/cli-args.test.ts | 1 + test/unit/jina.test.ts | 245 ++++++++++++++++++++++++++++ test/unit/providers-command.test.ts | 2 +- test/unit/read-command.test.ts | 151 +++++++++++++++++ test/unit/read.test.ts | 34 ++++ test/unit/resolve-async.test.ts | 2 +- test/unit/resolve.test.ts | 6 +- 26 files changed, 1150 insertions(+), 39 deletions(-) create mode 100644 src/commands/read.ts create mode 100644 src/core/read.ts create mode 100644 src/providers/jina.ts create mode 100644 test/unit/jina.test.ts create mode 100644 test/unit/read-command.test.ts create mode 100644 test/unit/read.test.ts diff --git a/README.md b/README.md index 1a84bb0..993f2c5 100644 --- a/README.md +++ b/README.md @@ -5,11 +5,11 @@ [![license](https://img.shields.io/github/license/oritwoen/askweb?style=flat&colorA=130f40&colorB=474787)](https://github.com/oritwoen/askweb/blob/main/LICENSE) [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/oritwoen/askweb) -One API for Brave, Exa, Tavily, SerpAPI, and SearXNG. Write your search logic once, swap the provider string, done. +One API for Brave, Exa, Jina, Tavily, SerpAPI, and SearXNG. Write your search logic once, swap the provider string, done. -If you're building an AI agent or a CLI tool that needs web search, you don't want to hardcode a single provider's API. They all return roughly the same thing, a list of URLs with titles and snippets, but the auth, endpoints, and response shapes are all different. Exa uses POST with `x-api-key`, Brave uses GET with `X-Subscription-Token`, Tavily puts the key in the request body. And so on. +If you're building an AI agent or a CLI tool that needs web search, you don't want to hardcode a single provider's API. They all return roughly the same thing, a list of URLs with titles and snippets, but the auth, endpoints, and response shapes are all different. Exa uses POST with `x-api-key`, Brave uses GET with `X-Subscription-Token`, Jina uses Bearer auth, Tavily puts the key in the request body. And so on. -`askweb` normalizes all of that behind a single interface. It also ships an [AI SDK](https://ai-sdk.dev/) tool and a CLI. +`askweb` normalizes all of that behind a single interface. It also ships an [AI SDK](https://ai-sdk.dev/) tool and a CLI. Search is query-to-results; read is URL-to-content. ## Install @@ -44,6 +44,7 @@ Swap the provider string, same code: ```typescript const brave = create('brave') // reads BRAVE_API_KEY +const jina = create('jina') // reads JINA_API_KEY const tavily = create('tavily') // reads TAVILY_API_KEY ``` @@ -79,34 +80,53 @@ const results = await searchAll('query', { }) ``` +### Read a URL + +Use `readUrl` when you already have a URL and want normalized page content: + +```typescript +import { readUrl } from 'askweb' + +const page = await readUrl('https://example.com/article', { + provider: 'jina', + format: 'markdown', + maxTokens: 4000, +}) + +console.log(page.title, page.content) +``` + +Jina read uses `r.jina.ai` and does not require an API key for basic reads; if `JINA_API_KEY` is present it is sent as Bearer auth. + ### AI SDK tool -The `askweb/ai` subpath exports a ready-made tool compatible with [Vercel AI SDK](https://ai-sdk.dev/docs/foundations/tools): +The `askweb/ai` subpath exports ready-made tools compatible with [Vercel AI SDK](https://ai-sdk.dev/docs/foundations/tools): ```typescript import { generateText } from 'ai' -import { searchTool } from 'askweb/ai' +import { readTool, searchTool } from 'askweb/ai' const { text } = await generateText({ model: yourModel, - tools: { webSearch: searchTool }, + tools: { webSearch: searchTool, webRead: readTool }, prompt: 'Find the latest TypeScript release notes', }) ``` -The tool accepts an optional `provider` parameter. Set it to `"all"` to query all available providers in parallel: +`searchTool` accepts an optional `provider` parameter. Set it to `"all"` to query all available providers in parallel. `readTool` accepts a URL and reads page content with a read-capable provider: ```typescript // The AI can choose: a specific provider, or "all" for parallel search -tools: { webSearch: searchTool } -// Input schema: { query: string, provider?: "brave" | "exa" | ... | "all", maxResults?: number } +tools: { webSearch: searchTool, webRead: readTool } +// searchTool input: { query: string, provider?: "brave" | "exa" | ... | "all", maxResults?: number } +// readTool input: { url: string, provider?: "jina", format?: "markdown" | "text" | "html" } ``` -When no provider is specified, the tool auto-detects the first available one from environment variables. +For `searchTool`, when no provider is specified, the tool auto-detects the first available one from environment variables. `readTool` defaults to Jina Reader. ### Pi extension -`askweb` ships with a [pi](https://pi.dev) extension that registers two tools and two commands. Install the package straight from GitHub: +`askweb` ships with a [pi](https://pi.dev) extension that registers three tools and two commands. Install the package straight from GitHub: ```bash pi install git:github.com/oritwoen/askweb @@ -115,6 +135,7 @@ pi install git:github.com/oritwoen/askweb Provided tools: - `askweb` — search the web with a single provider, or `provider="all"` to fan out across every configured/reachable provider in parallel +- `askweb_read` — read a URL into normalized content with a read-capable provider (currently Jina Reader) - `askweb_providers` — list built-in providers, env-var configuration, and reachability status Provided slash commands: @@ -122,7 +143,7 @@ Provided slash commands: - `/web [query]` — quick search from the TUI; results are shown as a selector and the chosen URL is pasted into the editor - `/web-providers` — show provider configuration and reachability status -The extension reuses the same env vars as the library (`EXA_API_KEY`, `BRAVE_API_KEY`, `TAVILY_API_KEY`, `SERPAPI_API_KEY`, or a self-hosted SearXNG). Pi bundles `@earendil-works/pi-coding-agent`, `@earendil-works/pi-tui`, and `typebox`, so no extra installs are needed. +The extension reuses the same env vars as the library (`EXA_API_KEY`, `BRAVE_API_KEY`, `JINA_API_KEY`, `TAVILY_API_KEY`, `SERPAPI_API_KEY`, or a self-hosted SearXNG). Pi bundles `@earendil-works/pi-coding-agent`, `@earendil-works/pi-tui`, and `typebox`, so no extra installs are needed. ## CLI @@ -130,6 +151,7 @@ The extension reuses the same env vars as the library (`EXA_API_KEY`, `BRAVE_API askweb "your query" askweb --provider brave "your query" --max-results 5 askweb search "your query" --json +askweb read https://example.com --format markdown --json askweb providers ``` @@ -137,12 +159,15 @@ askweb providers |---------|-------------| | `askweb ` | Search the web using the default provider | | `askweb search ` | Search the web using a provider | +| `askweb read ` | Read a URL into normalized content | | `askweb providers` | List built-in providers | | Flag | Description | |------|-------------| -| `--provider ` | Provider to use (default: `exa`) | -| `--max-results ` | Maximum results to return (default: `10`) | +| `--provider ` | Provider to use (search default: first configured; read default: `jina`) | +| `--max-results ` | Maximum search results to return (default: `10`) | +| `--format ` | Preferred read format | +| `--max-tokens ` | Maximum read tokens when supported | | `--json` | Output as JSON | ## Providers @@ -151,23 +176,25 @@ askweb providers |----------|---------|------|-----------| | Brave | `BRAVE_API_KEY` | Header | 2k queries/mo | | Exa | `EXA_API_KEY` | Header | 1k queries/mo | +| Jina | `JINA_API_KEY` | Bearer header | Required for search; optional for read | | SearXNG | - | None | Self-hosted | | SerpAPI | `SERPAPI_API_KEY` | Query param | 100 queries/mo | | Tavily | `TAVILY_API_KEY` | Body | 1k queries/mo | ### Result shape -All providers always return `{ url, title, snippet }`. Optional fields depend on what each provider's native API exposes — `askweb` passes them through without flattening: +All search providers always return `{ url, title, snippet }`. Optional fields depend on what each provider's native API exposes — `askweb` passes them through without flattening: | Provider | Optional fields populated | |----------|---------------------------| | Exa | `text` (full page), `highlights[]`, `summary` (AI), `score`, `publishedDate`, `author`, `image`, `favicon` | +| Jina | `text` (`content`/`text`), `publishedDate`, `image`, `metadata` | | Tavily | `text` (raw_content, full HTML/markdown), `score`, `publishedDate` | | Brave | `text` (joined `extra_snippets`), `favicon` | | SerpAPI | `image` (thumbnail), `publishedDate`, `favicon`, `metadata.{position, source, displayedLink}` | | SearXNG | `image`, `score`, `publishedDate`, `metadata.{engine, engines, category}` | -Pick the provider that fits the shape you want. Exa is closest to "AI search" (summary + highlights + full text on request). Tavily is best when you want the raw page content. Brave/SerpAPI/SearXNG are classic SERP-style metadata. +Pick the provider that fits the shape you want. Exa is closest to "AI search" (summary + highlights + full text on request). Jina uses Jina Search Foundation and can return result content plus metadata. Tavily is best when you want the raw page content. Brave/SerpAPI/SearXNG are classic SERP-style metadata. SearXNG requires no API key. It's a self-hosted metasearch engine. By default askweb connects to `http://localhost:8080`. Override with `baseURL`: @@ -197,13 +224,13 @@ try { } ``` -A 401 from Exa and a 401 from Brave both become `AuthError`. A 429 from any provider becomes `RateLimitError` with a `retryAfter` value. Everything else is `HTTPError` or the base `AskwebError`. +A 401 from any provider becomes `AuthError`. A 429 from any provider becomes `RateLimitError` with a `retryAfter` value. Everything else is `HTTPError` or the base `AskwebError`. For safety, `HTTPError.url` redacts sensitive query params and URL userinfo credentials before surfacing the URL in error messages. ## Data model -Every provider returns the same normalized type: +Every search provider returns the same normalized type: ```typescript interface SearchResult { @@ -222,7 +249,25 @@ interface SearchResult { } ``` -Optional fields depend on what the provider returns. Exa provides `score`, `text`, and `highlights`. Brave provides `favicon`. Not all providers populate all fields. +Optional fields depend on what the provider returns. Exa provides `score`, `text`, and `highlights`. Jina provides result `text`/metadata when available. Brave provides `favicon`. Not all providers populate all fields. + +Read results use the same naming for URL-to-content: + +```typescript +interface ReadResult { + url: string + title?: string + description?: string + content: string + text?: string + html?: string + publishedDate?: string + image?: string + links?: string[] | Record + images?: string[] | Record + metadata?: Record +} +``` Search options you can pass to `.search()` or `searchAll`: @@ -237,7 +282,20 @@ interface SearchOptions { } ``` -`maxResults` works with every provider. Domain filtering and date ranges are currently Exa-specific. `category` is supported by Exa and SearXNG. +`maxResults` works with every search provider. Domain filtering is supported by Exa, Tavily, and Jina include filters (`site`). Date ranges are currently Exa-specific. `category` is supported by Exa, Jina (`web`, `images`, `news`), and SearXNG. + +Read options you can pass to `readUrl`: + +```typescript +interface ReadOptions { + format?: 'markdown' | 'text' | 'html' + maxTokens?: number + targetSelector?: string + removeSelector?: string + timeout?: number + noCache?: boolean +} +``` ## Development diff --git a/package.json b/package.json index 6eec669..134613a 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "askweb", "version": "0.1.2", - "description": "Unified web search provider for agents and CLI.", + "description": "Unified web search and read provider for agents and CLI.", "type": "module", "license": "MIT", "sideEffects": [ diff --git a/packages/pi/extensions/askweb.ts b/packages/pi/extensions/askweb.ts index 9d93aa8..4a899d3 100644 --- a/packages/pi/extensions/askweb.ts +++ b/packages/pi/extensions/askweb.ts @@ -4,6 +4,8 @@ import { Type, type Static } from "typebox" import type { ProviderError, ProviderStatus, + ReadOptions, + ReadResult, SearchAllResult, SearchOptions, SearchResult, @@ -30,6 +32,14 @@ type SearchAllDetails = { type SearchDetails = SearchSingleDetails | SearchAllDetails +type ReadDetails = { + mode: "read" + url: string + provider: "jina" + options: ReadOptions + result: ReadResult +} + type AskwebModule = typeof import("askweb") let askwebModulePromise: Promise | undefined @@ -41,8 +51,10 @@ function loadAskweb(): Promise { return askwebModulePromise } -const PROVIDERS = ["auto", "all", "brave", "exa", "searxng", "serpapi", "tavily"] as const +const PROVIDERS = ["auto", "all", "brave", "exa", "jina", "searxng", "serpapi", "tavily"] as const const PROVIDER_HINT = `Provider to use. One of: ${PROVIDERS.join(", ")}. "auto" (or omit) picks the first available provider from env. Use "all" to query every configured provider in parallel.` +const READ_PROVIDERS = ["jina"] as const +const READ_PROVIDER_HINT = `Read provider to use. One of: ${READ_PROVIDERS.join(", ")}. Defaults to jina.` const MAX_RESULTS_HARD_CAP = 20 const DEFAULT_MAX_RESULTS = 10 @@ -84,24 +96,49 @@ const searchParameters = Type.Object({ ), }) +const readParameters = Type.Object({ + url: Type.String({ description: "URL to read." }), + provider: Type.Optional(Type.String({ description: READ_PROVIDER_HINT })), + format: Type.Optional( + Type.String({ description: 'Preferred content format: "markdown", "text", or "html".' }), + ), + maxTokens: Type.Optional( + Type.Number({ description: "Maximum tokens to return when supported.", minimum: 1 }), + ), + targetSelector: Type.Optional( + Type.String({ description: "CSS selector to target when supported." }), + ), + removeSelector: Type.Optional( + Type.String({ description: "CSS selector to remove when supported." }), + ), + timeout: Type.Optional( + Type.Number({ description: "Provider timeout in seconds when supported.", minimum: 1 }), + ), + noCache: Type.Optional( + Type.Boolean({ description: "Bypass provider cache when supported." }), + ), +}) + const emptyParameters = Type.Object({}) type SearchParams = Static +type ReadParams = Static type EmptyParams = Static type ProviderInput = (typeof PROVIDERS)[number] +type ReadProviderInput = (typeof READ_PROVIDERS)[number] export default function askwebExtension(pi: ExtensionAPI) { pi.registerTool({ name: "askweb", label: "Askweb Search", description: - "Search the web using one of the configured providers (Brave, Exa, Tavily, SerpAPI, SearXNG) or fan out to every available provider with provider=all. Always returns {url, title, snippet}; optional fields vary by provider: Exa adds summary/highlights/full text + score/author/image, Tavily adds full raw_content + score, Brave adds extra_snippets, SerpAPI adds thumbnail + position metadata, SearXNG adds engine metadata. Pick provider for the shape you need.", + "Search the web using one of the configured providers (Brave, Exa, Jina, Tavily, SerpAPI, SearXNG) or fan out to every available provider with provider=all. Always returns {url, title, snippet}; optional fields vary by provider: Exa adds summary/highlights/full text + score/author/image, Jina adds content/text + published date/image/metadata, Tavily adds full raw_content + score, Brave adds extra_snippets, SerpAPI adds thumbnail + position metadata, SearXNG adds engine metadata. Pick provider for the shape you need.", promptSnippet: "Search the web with askweb. Use provider=all to query every configured provider in parallel.", promptGuidelines: [ "Use askweb when the user explicitly asks for fresh web information, news, references, or links.", "Prefer a single provider when the user names one; use provider=all when freshness or coverage matters and at least two providers are configured.", - "For AI-style summaries/highlights/full page text prefer Exa; for raw full page content prefer Tavily; for classic SERP metadata Brave/SerpAPI/SearXNG are fine.", + "For AI-style summaries/highlights/full page text prefer Exa; for Jina Search Foundation results use Jina; for raw full page content prefer Tavily; for classic SERP metadata Brave/SerpAPI/SearXNG are fine.", "Pass maxResults conservatively (5-10) unless the user asks for more.", "Forward includeDomains/excludeDomains/startPublishedDate/endPublishedDate when the user gives concrete filters.", ], @@ -193,6 +230,59 @@ export default function askwebExtension(pi: ExtensionAPI) { }, }) + pi.registerTool({ + name: "askweb_read", + label: "Askweb Read", + description: + "Read-only/open-world network fetch: read a URL into normalized content using a read-capable provider. Defaults to Jina Reader (r.jina.ai). Returns URL, title/description when available, canonical content, and optional text/html/images/metadata.", + promptSnippet: "Read a URL with askweb_read when page content is needed, not just search results.", + promptGuidelines: [ + "Use askweb_read after search when the user needs the contents of a specific URL.", + "Use askweb for query-to-URL search; use askweb_read for URL-to-content reading.", + ], + parameters: readParameters, + renderCall(args, theme) { + return new Text(renderReadCall(args, theme), 0, 0) + }, + async execute(_toolCallId, params): Promise> { + const url = params.url.trim() + if (!url) { + throw new Error("URL cannot be empty") + } + + const rawProvider = (params.provider ?? "jina").trim() || "jina" + if (!isKnownReadProvider(rawProvider)) { + throw new Error( + `Unknown read provider "${rawProvider}". Available: ${READ_PROVIDERS.join(", ")}.`, + ) + } + + const format = normalizeReadFormat(params.format) + const readOptions: ReadOptions = stripUndefinedRead({ + format, + maxTokens: params.maxTokens, + targetSelector: params.targetSelector, + removeSelector: params.removeSelector, + timeout: params.timeout, + noCache: params.noCache, + }) + + const askweb = await loadAskweb() + const result = await askweb.readUrl(url, { provider: rawProvider, ...readOptions }) + const header = `[provider=${rawProvider}] read ${result.url}` + return { + content: [{ type: "text", text: withHeader(header, formatReadResult(result)) }], + details: { + mode: "read", + url, + provider: rawProvider, + options: readOptions, + result, + }, + } + }, + }) + pi.registerTool({ name: "askweb_providers", label: "Askweb Providers", @@ -311,6 +401,16 @@ function isKnownProvider(name: string): name is ProviderInput { return PROVIDERS.some((provider) => provider === name) } +function isKnownReadProvider(name: string): name is ReadProviderInput { + return READ_PROVIDERS.some((provider) => provider === name) +} + +function normalizeReadFormat(format: string | undefined): ReadOptions["format"] { + if (format === undefined || format === "") return undefined + if (format === "markdown" || format === "text" || format === "html") return format + throw new Error('Invalid read format. Expected "markdown", "text", or "html".') +} + function normalizeProvider(provider: ProviderInput | undefined): "all" | WebSearchProviderName | undefined { if (provider === "auto") { return undefined @@ -329,6 +429,17 @@ function stripUndefined(input: SearchOptions): SearchOptions { return out } +function stripUndefinedRead(input: ReadOptions): ReadOptions { + const out: ReadOptions = {} + if (input.format !== undefined) out.format = input.format + if (input.maxTokens !== undefined) out.maxTokens = input.maxTokens + if (input.targetSelector !== undefined) out.targetSelector = input.targetSelector + if (input.removeSelector !== undefined) out.removeSelector = input.removeSelector + if (input.timeout !== undefined) out.timeout = input.timeout + if (input.noCache !== undefined) out.noCache = input.noCache + return out +} + function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error) } @@ -392,6 +503,13 @@ function formatAllResults( return lines } +function formatReadResult(result: ReadResult): string[] { + const lines = [result.title || "(no title)", ` ${result.url}`] + if (result.description) lines.push(` ${truncateSingleLine(result.description, 160)}`) + if (result.content) lines.push("", result.content) + return lines +} + function renderSearchCall(params: SearchParams, theme: RenderTheme): string { const parts = [theme.fg("toolTitle", theme.bold("askweb"))] parts.push(theme.fg("dim", `"${truncateSingleLine(params.query, 120)}"`)) @@ -410,6 +528,16 @@ function renderSearchCall(params: SearchParams, theme: RenderTheme): string { return parts.join(" ") } +function renderReadCall(params: ReadParams, theme: RenderTheme): string { + const parts = [theme.fg("toolTitle", theme.bold("askweb_read"))] + parts.push(theme.fg("dim", truncateSingleLine(params.url, 120))) + if (params.provider) parts.push(theme.fg("muted", `provider=${params.provider}`)) + if (params.format) parts.push(theme.fg("muted", `format=${params.format}`)) + if (params.maxTokens !== undefined) + parts.push(theme.fg("muted", `maxTokens=${params.maxTokens}`)) + return parts.join(" ") +} + function truncateSingleLine(text: string, maxLength: number): string { const singleLine = text.replace(/\s+/g, " ").trim() return singleLine.length <= maxLength ? singleLine : `${singleLine.slice(0, maxLength - 1)}…` diff --git a/src/ai.ts b/src/ai.ts index 79b3dc0..44977ca 100644 --- a/src/ai.ts +++ b/src/ai.ts @@ -3,14 +3,16 @@ import { z } from 'zod' import { builtinProviders } from './core/providers.ts' import { create } from './core/registry.ts' import { searchAll } from './core/all.ts' -import { EmptyQueryError } from './core/errors.ts' +import { readUrl } from './core/read.ts' +import { EmptyQueryError, EmptyUrlError } from './core/errors.ts' import { resolveDefaultProvider, listProviders } from './core/resolve.ts' import './providers/index.ts' const providerNames = [...builtinProviders, 'all'] as const +const readProviderNames = ['jina'] as const export const searchTool = tool({ - description: 'Search the web using multiple search engines (Brave, Exa, Tavily, SerpAPI, SearXNG). Returns relevant web pages with titles, URLs, snippets, and optional metadata. Use provider "all" to query all available providers in parallel and get deduplicated results.', + description: 'Search the web using multiple search engines (Brave, Exa, Jina, Tavily, SerpAPI, SearXNG). Returns relevant web pages with titles, URLs, snippets, and optional metadata. Use provider "all" to query all available providers in parallel and get deduplicated results.', inputSchema: z.object({ query: z.string().describe('Search query'), provider: z.enum(providerNames).optional().describe('Provider to use. Defaults to first available from env. Use "all" for parallel search.'), @@ -37,6 +39,27 @@ export const searchTool = tool({ }, }) +export const readTool = tool({ + description: 'Read a URL into normalized content using a read-capable provider. Defaults to Jina Reader (r.jina.ai). Returns URL, title/description when available, canonical content, and optional text/html/images/metadata.', + inputSchema: z.object({ + url: z.string().describe('URL to read'), + provider: z.enum(readProviderNames).optional().describe('Read provider to use. Defaults to Jina.'), + format: z.enum(['markdown', 'text', 'html']).optional().describe('Preferred content format.'), + maxTokens: z.number().min(1).optional().describe('Maximum tokens to return when supported by the provider.'), + targetSelector: z.string().optional().describe('CSS selector to target when supported by the provider.'), + removeSelector: z.string().optional().describe('CSS selector to remove when supported by the provider.'), + timeout: z.number().min(1).optional().describe('Provider timeout in seconds when supported.'), + noCache: z.boolean().optional().describe('Bypass provider cache when supported.'), + }), + execute: async ({ url, provider, format, maxTokens, targetSelector, removeSelector, timeout, noCache }) => { + if (!url.trim()) { + throw new EmptyUrlError() + } + + return readUrl(url, { provider, format, maxTokens, targetSelector, removeSelector, timeout, noCache }) + }, +}) + export const providersTool = tool({ description: 'List available web search providers and their configuration status.', inputSchema: z.object({}), diff --git a/src/cli-args.ts b/src/cli-args.ts index b8ac1b4..eb4a5bd 100644 --- a/src/cli-args.ts +++ b/src/cli-args.ts @@ -1,5 +1,6 @@ const passthroughFirstArgs = new Set([ 'search', + 'read', 'providers', ]) diff --git a/src/cli.ts b/src/cli.ts index c1702d5..eac8ba7 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -8,10 +8,11 @@ const main = defineCommand({ meta: { name: 'askweb', version, - description: 'Unified web search provider for agents and CLI', + description: 'Unified web search and read provider for agents and CLI', }, subCommands: { search: () => import('./commands/search.ts').then(m => m.default), + read: () => import('./commands/read.ts').then(m => m.default), providers: () => import('./commands/providers.ts').then(m => m.default), }, }) diff --git a/src/commands/read.ts b/src/commands/read.ts new file mode 100644 index 0000000..0c79b5d --- /dev/null +++ b/src/commands/read.ts @@ -0,0 +1,146 @@ +import { defineCommand } from 'citty' +import { consola } from 'consola' + +export default defineCommand({ + meta: { + name: 'read', + description: 'Read a URL using a provider', + }, + args: { + url: { + type: 'positional', + description: 'URL to read', + required: true, + }, + provider: { + type: 'string', + description: 'Read provider to use', + default: 'jina', + }, + format: { + type: 'string', + description: 'Response format: markdown, text, or html', + }, + 'max-tokens': { + type: 'string', + description: 'Maximum tokens to return', + }, + json: { + type: 'boolean', + description: 'Output as JSON', + default: false, + }, + }, + async run({ args }) { + const { readUrl } = await import('../core/read.ts') + const { AuthError, UnknownProviderError, EmptyUrlError, ReadNotSupportedError } = await import('../core/errors.ts') + let providerName = args.provider || 'jina' + + try { + if (!args.url.trim()) { + consola.error('Read URL cannot be empty.') + process.exit(1) + } + + const format = parseFormat(args.format) + if (!format.ok) { + consola.error(format.message) + process.exit(1) + } + + const maxTokens = parseOptionalPositiveInt(args['max-tokens'], '--max-tokens') + if (!maxTokens.ok) { + consola.error(maxTokens.message) + process.exit(1) + } + + await import('../providers/index.ts') + providerName = args.provider || 'jina' + const result = await readUrl(args.url, { + provider: providerName, + format: format.value, + maxTokens: maxTokens.value, + }) + + if (args.json) { + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`) + return + } + + if (result.title) { + consola.log(`\x1b[1m\x1b[36m${result.title}\x1b[0m`) + } + consola.log(` ${result.url}`) + if (result.description) { + consola.log(` \x1b[90m${truncateSingleLine(result.description, 160)}\x1b[0m`) + } + consola.log('') + consola.log(result.content) + } + catch (error) { + if (error instanceof EmptyUrlError) { + consola.error('Read URL cannot be empty.') + process.exit(1) + } + if (error instanceof AuthError) { + const authProvider = providerName || error.provider + consola.error(`Authentication failed for provider "${authProvider}".`) + consola.info(`Set the ${authProvider.toUpperCase()}_API_KEY environment variable.`) + process.exit(1) + } + if (error instanceof UnknownProviderError) { + const { providers } = await import('../core/registry.ts') + consola.error(`Unknown provider: ${providerName}`) + const available = providers() + if (available.length > 0) { + consola.info(`Available providers: ${available.join(', ')}`) + } else { + consola.info('No providers registered. Import a provider first.') + } + process.exit(1) + } + if (error instanceof ReadNotSupportedError) { + consola.error(error.message) + process.exit(1) + } + throw error + } + }, +}) + +type ParsedOptionalNumber = + | { ok: true; value: number | undefined } + | { ok: false; message: string } + +type ParsedFormat = + | { ok: true; value: 'markdown' | 'text' | 'html' | undefined } + | { ok: false; message: string } + +function parseOptionalPositiveInt(input: string | undefined, flagName: string): ParsedOptionalNumber { + if (input == null || input === '') { + return { ok: true, value: undefined } + } + if (!/^\d+$/.test(input)) { + return { ok: false, message: `Invalid ${flagName} value. Expected a positive integer.` } + } + const value = Number.parseInt(input, 10) + if (value < 1) { + return { ok: false, message: `Invalid ${flagName} value. Expected a positive integer.` } + } + return { ok: true, value } +} + +function parseFormat(input: string | undefined): ParsedFormat { + if (input == null || input === '') { + return { ok: true, value: undefined } + } + if (input === 'markdown' || input === 'text' || input === 'html') { + return { ok: true, value: input } + } + return { ok: false, message: 'Invalid --format value. Expected markdown, text, or html.' } +} + +function truncateSingleLine(text: string, maxLength: number): string { + const singleLine = text.replace(/\s+/g, ' ').trim() + return singleLine.length <= maxLength ? singleLine : `${singleLine.slice(0, maxLength - 1)}…` +} diff --git a/src/core/errors.ts b/src/core/errors.ts index d74a971..b87e106 100644 --- a/src/core/errors.ts +++ b/src/core/errors.ts @@ -74,6 +74,25 @@ export class EmptyQueryError extends AskwebError { } } +/** Thrown when the read URL is empty or whitespace-only. */ +export class EmptyUrlError extends AskwebError { + constructor() { + super('Read URL cannot be empty') + this.name = 'EmptyUrlError' + } +} + +/** Thrown when a provider does not implement the read capability. */ +export class ReadNotSupportedError extends AskwebError { + readonly provider: string + + constructor(provider: string) { + super(`Provider does not support read: ${provider}`) + this.name = 'ReadNotSupportedError' + this.provider = provider + } +} + /** Thrown when no provider can be selected from env or registry. */ export class NoProviderConfiguredError extends AskwebError { constructor() { diff --git a/src/core/providers.ts b/src/core/providers.ts index 063477f..013c442 100644 --- a/src/core/providers.ts +++ b/src/core/providers.ts @@ -1,6 +1,7 @@ export const builtinProviders = [ 'brave', 'exa', + 'jina', 'searxng', 'serpapi', 'tavily', diff --git a/src/core/read.ts b/src/core/read.ts new file mode 100644 index 0000000..f4251b0 --- /dev/null +++ b/src/core/read.ts @@ -0,0 +1,25 @@ +import type { ReadOptions, ReadResult } from './types.ts' +import { EmptyUrlError, ReadNotSupportedError } from './errors.ts' +import { create } from './registry.ts' + +export interface ReadUrlOptions extends ReadOptions { + provider?: string +} + +const DEFAULT_READ_PROVIDER = 'jina' + +export async function readUrl(url: string, options?: ReadUrlOptions): Promise { + const trimmedUrl = url.trim() + if (!trimmedUrl) { + throw new EmptyUrlError() + } + + const { provider: requestedProvider, ...readOptions } = options ?? {} + const providerName = requestedProvider?.trim() || DEFAULT_READ_PROVIDER + const provider = create(providerName) + if (typeof provider.read !== 'function') { + throw new ReadNotSupportedError(providerName) + } + + return provider.read(trimmedUrl, readOptions) +} diff --git a/src/core/resolve.ts b/src/core/resolve.ts index 6d7d2ed..c6df4b0 100644 --- a/src/core/resolve.ts +++ b/src/core/resolve.ts @@ -5,6 +5,7 @@ import { NoProviderAvailableError, NoProviderConfiguredError } from './errors.ts const envKeys: Record = { EXA_API_KEY: 'exa', BRAVE_API_KEY: 'brave', + JINA_API_KEY: 'jina', TAVILY_API_KEY: 'tavily', SERPAPI_API_KEY: 'serpapi', } diff --git a/src/core/types.ts b/src/core/types.ts index 3f07525..2d06ff3 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -22,9 +22,33 @@ export interface SearchOptions { category?: string } +export interface ReadResult { + url: string + title?: string + description?: string + content: string + text?: string + html?: string + publishedDate?: string + image?: string + links?: string[] | Record + images?: string[] | Record + metadata?: Record +} + +export interface ReadOptions { + format?: 'markdown' | 'text' | 'html' + maxTokens?: number + targetSelector?: string + removeSelector?: string + timeout?: number + noCache?: boolean +} + export interface SearchProvider { name(): string search(query: string, options?: SearchOptions): Promise + read?(url: string, options?: ReadOptions): Promise /** * Optional reachability probe. Used by {@link searchAll} and async detection * helpers to skip self-hosted / optional providers whose endpoint is not @@ -38,6 +62,7 @@ export interface SearchProvider { export interface ProviderConfig { apiKey?: string baseURL?: string + readBaseURL?: string } export type ProviderFactory = (config: ProviderConfig) => SearchProvider diff --git a/src/index.ts b/src/index.ts index 9016bd7..0474069 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,9 +4,9 @@ export { version } from './version.ts' export { builtinProviders, type WebSearchProviderName } from './core/providers.ts' -export type { SearchResult, SearchOptions, SearchProvider, ProviderConfig, ProviderFactory, ClientOptions } from './core/types.ts' +export type { SearchResult, SearchOptions, ReadResult, ReadOptions, SearchProvider, ProviderConfig, ProviderFactory, ClientOptions } from './core/types.ts' -export { AskwebError, HTTPError, AuthError, RateLimitError, UnknownProviderError, NoProviderConfiguredError, NoProviderAvailableError, EmptyQueryError, InvalidDateFilterError, normalizeError, validateDateFilters } from './core/errors.ts' +export { AskwebError, HTTPError, AuthError, RateLimitError, UnknownProviderError, NoProviderConfiguredError, NoProviderAvailableError, EmptyQueryError, EmptyUrlError, ReadNotSupportedError, InvalidDateFilterError, normalizeError, validateDateFilters } from './core/errors.ts' export { Client, defaultClient } from './core/client.ts' @@ -15,6 +15,9 @@ export { register, create, providers, has } from './core/registry.ts' export { searchAll, searchAllDetailed } from './core/all.ts' export type { SearchAllOptions, SearchAllResult, SearchAllResponse, ProviderError } from './core/all.ts' +export { readUrl } from './core/read.ts' +export type { ReadUrlOptions } from './core/read.ts' + export { resolveDefaultProvider, resolveDefaultProviderAsync, diff --git a/src/opencode.ts b/src/opencode.ts index b37a19f..9f5d9f9 100644 --- a/src/opencode.ts +++ b/src/opencode.ts @@ -4,16 +4,18 @@ import { encode } from '@toon-format/toon' import { builtinProviders } from './core/providers.ts' import { create } from './core/registry.ts' import { searchAll } from './core/all.ts' +import { readUrl } from './core/read.ts' import { resolveDefaultProvider, listProviders } from './core/resolve.ts' import './providers/index.ts' const z = tool.schema const providerNames = [...builtinProviders, 'all'] as const +const readProviderNames = ['jina'] as const const AskwebPlugin: Plugin = async () => ({ tool: { askweb: tool({ - description: 'Search the web using multiple search engines (Brave, Exa, Tavily, SerpAPI, SearXNG). Returns relevant web pages with titles, URLs, snippets, and optional metadata. Use provider "all" to query all available providers in parallel and get deduplicated results.', + description: 'Search the web using multiple search engines (Brave, Exa, Jina, Tavily, SerpAPI, SearXNG). Returns relevant web pages with titles, URLs, snippets, and optional metadata. Use provider "all" to query all available providers in parallel and get deduplicated results.', args: { query: z.string().describe('Search query'), provider: z.enum(providerNames).optional().describe('Provider to use. Defaults to first available from env. Use "all" for parallel search.'), @@ -30,6 +32,19 @@ const AskwebPlugin: Plugin = async () => ({ return encode(await create(name).search(query, { maxResults })) }, }), + askweb_read: tool({ + description: 'Read a URL into normalized content using a read-capable provider. Defaults to Jina Reader (r.jina.ai).', + args: { + url: z.string().describe('URL to read'), + provider: z.enum(readProviderNames).optional().describe('Read provider to use. Defaults to Jina.'), + format: z.enum(['markdown', 'text', 'html']).optional().describe('Preferred content format.'), + maxTokens: z.number().min(1).optional().describe('Maximum tokens to return when supported by the provider.'), + }, + async execute(args) { + const { url, provider, format, maxTokens } = args + return encode(await readUrl(url, { provider, format, maxTokens })) + }, + }), askweb_providers: tool({ description: 'List available web search providers and their configuration status.', args: {}, diff --git a/src/providers/index.ts b/src/providers/index.ts index d09b057..ef367b2 100644 --- a/src/providers/index.ts +++ b/src/providers/index.ts @@ -1,5 +1,6 @@ import './exa.ts' import './brave.ts' +import './jina.ts' import './tavily.ts' import './serpapi.ts' import './searxng.ts' diff --git a/src/providers/jina.ts b/src/providers/jina.ts new file mode 100644 index 0000000..b51e094 --- /dev/null +++ b/src/providers/jina.ts @@ -0,0 +1,184 @@ +import type { SearchResult, SearchOptions, ReadResult, ReadOptions, SearchProvider, ProviderConfig, ProviderFactory } from '../core/types.ts' +import { defaultClient } from '../core/client.ts' +import type { Client } from '../core/client.ts' +import { AuthError, normalizeError } from '../core/errors.ts' +import { register } from '../core/registry.ts' + +interface JinaResult { + title?: string + description?: string + url: string + content?: string + text?: string + html?: string + publishedTime?: string + links?: string[] | Record | null + images?: string[] | Record | null + metadata?: Record | null + warning?: string + screenshotUrl?: string + pageshotUrl?: string +} + +interface JinaSearchResponse { + code: number + status: number + data?: JinaResult[] | null + meta?: Record +} + +interface JinaReadResponse { + code: number + status: number + data?: JinaResult | null + meta?: Record +} + +const JINA_MAX_RESULTS = 20 + +class JinaProvider implements SearchProvider { + private readonly client: Client + private readonly searchBaseURL: string + private readonly readBaseURL: string + private readonly apiKey?: string + + constructor(config: ProviderConfig) { + this.client = defaultClient() + this.searchBaseURL = (config.baseURL ?? 'https://s.jina.ai').replace(/\/+$/, '') + this.readBaseURL = (config.readBaseURL ?? deriveReadBaseURL(this.searchBaseURL)).replace(/\/+$/, '') + this.apiKey = config.apiKey + } + + name(): string { + return 'jina' + } + + async search(query: string, options?: SearchOptions): Promise { + if (!this.apiKey) { + throw new AuthError('Missing API key for Jina. Set JINA_API_KEY', 'jina') + } + + try { + const params = new URLSearchParams({ + q: query, + count: String(clampMaxResults(options?.maxResults ?? 10)), + }) + + if (isJinaSearchType(options?.category)) { + params.set('type', options.category) + } + + for (const domain of options?.includeDomains ?? []) { + params.append('site', domain) + } + + const url = `${this.searchBaseURL}/search?${params.toString()}` + const headers = { + 'Authorization': `Bearer ${this.apiKey}`, + 'Accept': 'application/json', + } + const response = await this.client.getJSON(url, headers) + return (response.data ?? []).map(mapSearchResult) + } + catch (error) { + throw normalizeError(error, 'jina') + } + } + + async read(url: string, options?: ReadOptions): Promise { + try { + const requestUrl = `${this.readBaseURL}/${encodeURIComponent(url)}` + const response = await this.client.getJSON(requestUrl, readHeaders(this.apiKey, options)) + return mapReadResult(response.data ?? { url, content: '' }) + } + catch (error) { + throw normalizeError(error, 'jina') + } + } +} + +function deriveReadBaseURL(searchBaseURL: string): string { + return searchBaseURL === 'https://s.jina.ai' ? 'https://r.jina.ai' : searchBaseURL +} + +function readHeaders(apiKey: string | undefined, options: ReadOptions | undefined): Record { + const headers: Record = { Accept: 'application/json' } + + if (apiKey) headers.Authorization = `Bearer ${apiKey}` + if (options?.format) headers['X-Respond-With'] = options.format + if (options?.maxTokens !== undefined) headers['X-Max-Tokens'] = String(options.maxTokens) + if (options?.targetSelector) headers['X-Target-Selector'] = options.targetSelector + if (options?.removeSelector) headers['X-Remove-Selector'] = options.removeSelector + if (options?.timeout !== undefined) headers['X-Timeout'] = String(options.timeout) + if (options?.noCache) headers['X-No-Cache'] = 'true' + + return headers +} + +function clampMaxResults(maxResults: number): number { + return Math.min(Math.max(maxResults, 1), JINA_MAX_RESULTS) +} + +function isJinaSearchType(category: string | undefined): category is 'web' | 'images' | 'news' { + return category === 'web' || category === 'images' || category === 'news' +} + +function mapSearchResult(result: JinaResult): SearchResult { + return { + url: result.url, + title: result.title ?? '', + snippet: result.description ?? snippetFrom(result.content) ?? snippetFrom(result.text) ?? '', + publishedDate: result.publishedTime, + text: result.content ?? result.text, + image: firstImage(result.images), + metadata: resultMetadata(result), + } +} + +function mapReadResult(result: JinaResult): ReadResult { + return { + url: result.url, + title: result.title, + description: result.description, + content: result.content ?? result.text ?? result.html ?? '', + text: result.text, + html: result.html, + publishedDate: result.publishedTime, + image: firstImage(result.images), + links: result.links ?? undefined, + images: result.images ?? undefined, + metadata: resultMetadata(result), + } +} + +function snippetFrom(text: string | undefined): string | undefined { + return text ? text.slice(0, 200) : undefined +} + +function firstImage(images: JinaResult['images']): string | undefined { + if (Array.isArray(images)) { + return images.find(isNonEmptyString) + } + if (images && typeof images === 'object') { + return Object.values(images).find(isNonEmptyString) + } + return undefined +} + +function isNonEmptyString(value: unknown): value is string { + return typeof value === 'string' && value.length > 0 +} + +function resultMetadata(result: JinaResult): Record | undefined { + const metadata: Record = result.metadata ? { ...result.metadata } : {} + + if (result.warning) metadata.warning = result.warning + if (result.screenshotUrl) metadata.screenshotUrl = result.screenshotUrl + if (result.pageshotUrl) metadata.pageshotUrl = result.pageshotUrl + + return Object.keys(metadata).length > 0 ? metadata : undefined +} + +const factory: ProviderFactory = (config) => new JinaProvider(config) + +register('jina', 'https://s.jina.ai', factory) diff --git a/test/index.test.ts b/test/index.test.ts index 8e09c37..e0c5a2a 100644 --- a/test/index.test.ts +++ b/test/index.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest' -import { builtinProviders, create, version } from '../src/index.ts' +import { builtinProviders, create, readUrl, version } from '../src/index.ts' describe('askweb', () => { it('should export version matching package.json', () => { @@ -7,13 +7,17 @@ describe('askweb', () => { }) it('should list all built-in provider names', () => { - expect(builtinProviders).toEqual(['brave', 'exa', 'searxng', 'serpapi', 'tavily']) + expect(builtinProviders).toEqual(['brave', 'exa', 'jina', 'searxng', 'serpapi', 'tavily']) }) it('should register built-in providers from main entrypoint', () => { for (const provider of builtinProviders) { - const config = provider === 'searxng' ? undefined : { apiKey: 'test-api-key' } + const config = provider === 'searxng' || provider === 'jina' ? undefined : { apiKey: 'test-api-key' } expect(() => create(provider, config)).not.toThrow() } }) + + it('should export readUrl', () => { + expect(readUrl).toBeTypeOf('function') + }) }) diff --git a/test/unit/ai-tool.test.ts b/test/unit/ai-tool.test.ts index 136b1f2..e9d9409 100644 --- a/test/unit/ai-tool.test.ts +++ b/test/unit/ai-tool.test.ts @@ -15,8 +15,8 @@ vi.mock('../../src/core/client.ts', () => ({ })), })) -import { searchTool } from '../../src/ai.ts' -import { EmptyQueryError } from '../../src/core/errors.ts' +import { readTool, searchTool } from '../../src/ai.ts' +import { EmptyQueryError, EmptyUrlError } from '../../src/core/errors.ts' const exaResponse = { requestId: 'test-req', @@ -62,7 +62,7 @@ const searxngResponse = { } const savedEnv: Record = {} -const envKeys = ['EXA_API_KEY', 'BRAVE_API_KEY', 'TAVILY_API_KEY', 'SERPAPI_API_KEY'] +const envKeys = ['EXA_API_KEY', 'BRAVE_API_KEY', 'JINA_API_KEY', 'TAVILY_API_KEY', 'SERPAPI_API_KEY'] describe('searchTool', () => { beforeEach(() => { @@ -250,3 +250,42 @@ describe('searchTool', () => { expect(results[0]).toHaveProperty('title') }) }) + +describe('readTool', () => { + beforeEach(() => { + mockGetJSON.mockReset() + }) + + it('reads a URL with Jina by default', async () => { + mockGetJSON.mockResolvedValueOnce({ + code: 200, + status: 20000, + data: { + title: 'Read Result', + url: 'https://example.com/', + content: 'Read content', + }, + }) + + const result = await readTool.execute!( + { url: 'https://example.com', format: 'markdown' }, + { toolCallId: 'read-call-1', messages: [] }, + ) + + expect(result.content).toBe('Read content') + const [url, headers] = mockGetJSON.mock.calls[0] + expect(url).toBe('https://r.jina.ai/https%3A%2F%2Fexample.com') + expect(headers).toEqual({ Accept: 'application/json', 'X-Respond-With': 'markdown' }) + }) + + it('rejects empty URL', async () => { + await expect( + readTool.execute!( + { url: ' ' }, + { toolCallId: 'read-empty', messages: [] }, + ), + ).rejects.toThrow(EmptyUrlError) + + expect(mockGetJSON).not.toHaveBeenCalled() + }) +}) diff --git a/test/unit/all.test.ts b/test/unit/all.test.ts index 0c8fd07..1131340 100644 --- a/test/unit/all.test.ts +++ b/test/unit/all.test.ts @@ -50,6 +50,7 @@ describe('searchAll', () => { mockGetJSON.mockReset() delete process.env.EXA_API_KEY delete process.env.BRAVE_API_KEY + delete process.env.JINA_API_KEY delete process.env.TAVILY_API_KEY delete process.env.SERPAPI_API_KEY }) @@ -373,6 +374,7 @@ describe('searchAllDetailed', () => { mockGetJSON.mockReset() delete process.env.EXA_API_KEY delete process.env.BRAVE_API_KEY + delete process.env.JINA_API_KEY delete process.env.TAVILY_API_KEY delete process.env.SERPAPI_API_KEY }) diff --git a/test/unit/cli-args.test.ts b/test/unit/cli-args.test.ts index c553153..04f8f92 100644 --- a/test/unit/cli-args.test.ts +++ b/test/unit/cli-args.test.ts @@ -4,6 +4,7 @@ import { normalizeMainArgs } from '../../src/cli-args.ts' describe('CLI main args', () => { it('keeps explicit subcommands unchanged', () => { expect(normalizeMainArgs(['search', 'query'])).toEqual(['search', 'query']) + expect(normalizeMainArgs(['read', 'https://example.com'])).toEqual(['read', 'https://example.com']) expect(normalizeMainArgs(['providers'])).toEqual(['providers']) }) diff --git a/test/unit/jina.test.ts b/test/unit/jina.test.ts new file mode 100644 index 0000000..b920351 --- /dev/null +++ b/test/unit/jina.test.ts @@ -0,0 +1,245 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' + +const mockGetJSON = vi.fn() + +vi.mock('../../src/core/client.ts', () => ({ + Client: vi.fn(), + defaultClient: vi.fn(() => ({ + getJSON: mockGetJSON, + postJSON: vi.fn(), + maxRetries: 5, + baseDelay: 50, + timeout: 30000, + userAgent: 'askweb/0.0.1', + })), +})) + +import { create, has } from '../../src/core/registry.ts' +import { AuthError } from '../../src/core/errors.ts' +import type { SearchResult, ReadResult } from '../../src/core/types.ts' + +// Triggers self-registration of jina provider +import '../../src/providers/index.ts' + +const jinaResponse = { + code: 200, + status: 20000, + data: [{ + title: 'Test Result', + url: 'https://example.com', + description: 'A test description from Jina search', + content: 'Full content from Jina search result', + publishedTime: '2024-07-01T00:00:00Z', + images: ['https://example.com/image.png'], + metadata: { source: 'jina' }, + warning: 'partial result', + }], +} + +describe('jina provider', () => { + beforeEach(() => { + mockGetJSON.mockReset() + mockGetJSON.mockResolvedValue(jinaResponse) + delete process.env.JINA_API_KEY + }) + + describe('self-registration', () => { + it('registers itself on import', () => { + expect(has('jina')).toBe(true) + }) + }) + + describe('create', () => { + it('creates provider with apiKey', () => { + expect(() => create('jina', { apiKey: 'test-key' })).not.toThrow() + }) + + it('creates provider without apiKey for read-only use', () => { + expect(() => create('jina', {})).not.toThrow() + }) + }) + + describe('name()', () => { + it('returns jina', () => { + const provider = create('jina', { apiKey: 'test-key' }) + expect(provider.name()).toBe('jina') + }) + }) + + describe('search()', () => { + it('throws AuthError without apiKey and without env var', async () => { + const provider = create('jina', {}) + await expect(provider.search('test query')).rejects.toThrow(AuthError) + expect(mockGetJSON).not.toHaveBeenCalled() + }) + + it('calls getJSON with correct URL and bearer auth headers', async () => { + const provider = create('jina', { apiKey: 'test-key' }) + await provider.search('test query') + + expect(mockGetJSON).toHaveBeenCalledOnce() + const [url, headers] = mockGetJSON.mock.calls[0] + + expect(url).toContain('https://s.jina.ai/search?') + expect(url).toContain('q=test+query') + expect(url).toContain('count=10') + expect(headers).toEqual({ + Authorization: 'Bearer test-key', + Accept: 'application/json', + }) + }) + + it('maps result fields correctly', async () => { + const provider = create('jina', { apiKey: 'test-key' }) + const results: SearchResult[] = await provider.search('test query') + + expect(results).toHaveLength(1) + const result = results[0] + expect(result.url).toBe('https://example.com') + expect(result.title).toBe('Test Result') + expect(result.snippet).toBe('A test description from Jina search') + expect(result.text).toBe('Full content from Jina search result') + expect(result.publishedDate).toBe('2024-07-01T00:00:00Z') + expect(result.image).toBe('https://example.com/image.png') + expect(result.metadata).toEqual({ source: 'jina', warning: 'partial result' }) + }) + + it('maps maxResults to count query param and clamps to Jina limit', async () => { + const provider = create('jina', { apiKey: 'test-key' }) + await provider.search('test query', { maxResults: 25 }) + + const [url] = mockGetJSON.mock.calls[0] + expect(url).toContain('count=20') + }) + + it('maps includeDomains and news category to Jina query params', async () => { + const provider = create('jina', { apiKey: 'test-key' }) + await provider.search('test query', { includeDomains: ['example.com'], category: 'news' }) + + const [url] = mockGetJSON.mock.calls[0] + expect(url).toContain('site=example.com') + expect(url).toContain('type=news') + }) + + it('falls back to content for snippet when description is missing', async () => { + mockGetJSON.mockResolvedValueOnce({ + code: 200, + status: 20000, + data: [{ + url: 'https://example.com', + content: 'A'.repeat(300), + }], + }) + + const provider = create('jina', { apiKey: 'test-key' }) + const results = await provider.search('query') + + expect(results[0].snippet).toBe('A'.repeat(200)) + expect(results[0].title).toBe('') + }) + + it('returns empty array when data is undefined', async () => { + mockGetJSON.mockResolvedValueOnce({ + code: 200, + status: 20000, + data: undefined, + }) + + const provider = create('jina', { apiKey: 'test-key' }) + const results = await provider.search('query') + + expect(results).toEqual([]) + }) + }) + + describe('read()', () => { + it('calls r.jina.ai with encoded URL and JSON accept header without requiring apiKey', async () => { + mockGetJSON.mockResolvedValueOnce({ + code: 200, + status: 20000, + data: { + title: 'Read Result', + description: 'Read description', + url: 'https://example.com/', + content: 'Markdown content', + }, + }) + + const provider = create('jina', {}) + const result = await provider.read!('https://example.com/?a=1&b=2') + + expect(mockGetJSON).toHaveBeenCalledOnce() + const [url, headers] = mockGetJSON.mock.calls[0] + expect(url).toBe('https://r.jina.ai/https%3A%2F%2Fexample.com%2F%3Fa%3D1%26b%3D2') + expect(headers).toEqual({ Accept: 'application/json' }) + expect(result.content).toBe('Markdown content') + }) + + it('passes read options as Jina Reader headers', async () => { + mockGetJSON.mockResolvedValueOnce({ + code: 200, + status: 20000, + data: { url: 'https://example.com/', content: 'Text content' }, + }) + + const provider = create('jina', { apiKey: 'test-key' }) + await provider.read!('https://example.com', { + format: 'text', + maxTokens: 500, + targetSelector: 'main', + removeSelector: 'nav', + timeout: 30, + noCache: true, + }) + + const [, headers] = mockGetJSON.mock.calls[0] + expect(headers).toEqual({ + Accept: 'application/json', + Authorization: 'Bearer test-key', + 'X-Respond-With': 'text', + 'X-Max-Tokens': '500', + 'X-Target-Selector': 'main', + 'X-Remove-Selector': 'nav', + 'X-Timeout': '30', + 'X-No-Cache': 'true', + }) + }) + + it('maps read result fields correctly', async () => { + mockGetJSON.mockResolvedValueOnce({ + code: 200, + status: 20000, + data: { + title: 'Read Result', + description: 'Read description', + url: 'https://example.com/', + content: 'Markdown content', + text: 'Plain content', + html: '
HTML content
', + publishedTime: '2024-08-01T00:00:00Z', + links: ['https://example.com/a'], + images: { hero: 'https://example.com/hero.png' }, + metadata: { lang: 'en' }, + warning: 'cached', + }, + }) + + const provider = create('jina', {}) + const result: ReadResult = await provider.read!('https://example.com') + + expect(result).toEqual({ + url: 'https://example.com/', + title: 'Read Result', + description: 'Read description', + content: 'Markdown content', + text: 'Plain content', + html: '
HTML content
', + publishedDate: '2024-08-01T00:00:00Z', + image: 'https://example.com/hero.png', + links: ['https://example.com/a'], + images: { hero: 'https://example.com/hero.png' }, + metadata: { lang: 'en', warning: 'cached' }, + }) + }) + }) +}) diff --git a/test/unit/providers-command.test.ts b/test/unit/providers-command.test.ts index 0c242e1..d764b8f 100644 --- a/test/unit/providers-command.test.ts +++ b/test/unit/providers-command.test.ts @@ -11,7 +11,7 @@ vi.mock('consola', () => ({ import providersCommand from '../../src/commands/providers.ts' import { builtinProviders } from '../../src/index.ts' -const envKeys = ['EXA_API_KEY', 'BRAVE_API_KEY', 'TAVILY_API_KEY', 'SERPAPI_API_KEY'] +const envKeys = ['EXA_API_KEY', 'BRAVE_API_KEY', 'JINA_API_KEY', 'TAVILY_API_KEY', 'SERPAPI_API_KEY'] describe('providers command', () => { const savedEnv: Record = {} diff --git a/test/unit/read-command.test.ts b/test/unit/read-command.test.ts new file mode 100644 index 0000000..23de08d --- /dev/null +++ b/test/unit/read-command.test.ts @@ -0,0 +1,151 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { EmptyUrlError, ReadNotSupportedError } from '../../src/core/errors.ts' + +const mockLog = vi.fn() +const mockInfo = vi.fn() +const mockError = vi.fn() +const mockReadUrl = vi.fn() + +vi.mock('consola', () => ({ + consola: { + log: (...args: unknown[]) => mockLog(...args), + info: (...args: unknown[]) => mockInfo(...args), + error: (...args: unknown[]) => mockError(...args), + }, +})) + +vi.mock('../../src/core/read.ts', () => ({ + readUrl: (...args: unknown[]) => mockReadUrl(...args), +})) + +vi.mock('../../src/core/registry.ts', () => ({ + providers: vi.fn(() => ['jina']), +})) + +vi.mock('../../src/providers/index.ts', () => ({})) + +import readCommand from '../../src/commands/read.ts' + +type ReadRunInput = Parameters>[0] +type ReadRunArgs = { + _: string[] + url: string + provider?: string + format?: string + 'max-tokens'?: string + json: boolean + [key: string]: string | number | boolean | string[] | undefined +} + +const defaultArgs: ReadRunArgs = { + _: [], + url: 'https://example.com', + provider: 'jina', + json: false, +} + +function makeArgs(overrides: Partial = {}): ReadRunArgs { + return { ...defaultArgs, ...overrides } +} + +function runRead(overrides: Partial = {}) { + const context = { + args: makeArgs(overrides), + rawArgs: [], + cmd: readCommand, + } as ReadRunInput + return readCommand.run!(context) +} + +describe('read command', () => { + let exitSpy: ReturnType + let writeSpy: ReturnType + + beforeEach(() => { + mockLog.mockReset() + mockInfo.mockReset() + mockError.mockReset() + mockReadUrl.mockReset() + mockReadUrl.mockResolvedValue({ + url: 'https://example.com', + title: 'Example', + description: 'Example description', + content: 'Example content', + }) + exitSpy = vi.spyOn(process, 'exit').mockImplementation((() => { + throw new Error('__EXIT__') + }) as never) + writeSpy = vi.spyOn(process.stdout, 'write').mockImplementation(() => true) + }) + + afterEach(() => { + exitSpy.mockRestore() + writeSpy.mockRestore() + }) + + it('uses jina provider by default', async () => { + await runRead({ provider: undefined }) + + expect(mockReadUrl).toHaveBeenCalledWith('https://example.com', { + provider: 'jina', + format: undefined, + maxTokens: undefined, + }) + }) + + it('passes format and max tokens', async () => { + await runRead({ format: 'text', 'max-tokens': '500' }) + + expect(mockReadUrl).toHaveBeenCalledWith('https://example.com', { + provider: 'jina', + format: 'text', + maxTokens: 500, + }) + }) + + it('outputs JSON when --json is set', async () => { + await runRead({ json: true }) + + expect(writeSpy).toHaveBeenCalledOnce() + const parsed = JSON.parse(String(writeSpy.mock.calls[0][0])) + expect(parsed.content).toBe('Example content') + }) + + it('exits with a helpful message for empty URL', async () => { + await expect(runRead({ url: ' ' })).rejects.toThrow('__EXIT__') + + expect(mockError).toHaveBeenCalledWith('Read URL cannot be empty.') + expect(mockReadUrl).not.toHaveBeenCalled() + expect(exitSpy).toHaveBeenCalledWith(1) + }) + + it('exits with a helpful message for invalid format', async () => { + await expect(runRead({ format: 'pdf' })).rejects.toThrow('__EXIT__') + + expect(mockError).toHaveBeenCalledWith('Invalid --format value. Expected markdown, text, or html.') + expect(mockReadUrl).not.toHaveBeenCalled() + }) + + it('exits with a helpful message for invalid max tokens', async () => { + await expect(runRead({ 'max-tokens': '0' })).rejects.toThrow('__EXIT__') + + expect(mockError).toHaveBeenCalledWith('Invalid --max-tokens value. Expected a positive integer.') + expect(mockReadUrl).not.toHaveBeenCalled() + }) + + it('handles EmptyUrlError from core', async () => { + mockReadUrl.mockRejectedValueOnce(new EmptyUrlError()) + + await expect(runRead()).rejects.toThrow('__EXIT__') + + expect(mockError).toHaveBeenCalledWith('Read URL cannot be empty.') + }) + + it('handles ReadNotSupportedError from core', async () => { + mockReadUrl.mockRejectedValueOnce(new ReadNotSupportedError('brave')) + + await expect(runRead({ provider: 'brave' })).rejects.toThrow('__EXIT__') + + expect(mockError).toHaveBeenCalledWith('Provider does not support read: brave') + }) +}) diff --git a/test/unit/read.test.ts b/test/unit/read.test.ts new file mode 100644 index 0000000..c0ad2f6 --- /dev/null +++ b/test/unit/read.test.ts @@ -0,0 +1,34 @@ +import { describe, it, expect, vi } from 'vitest' +import { readUrl } from '../../src/core/read.ts' +import { register } from '../../src/core/registry.ts' +import { EmptyUrlError, ReadNotSupportedError } from '../../src/core/errors.ts' + +describe('readUrl', () => { + it('passes explicit provider and read options through', async () => { + const providerName = `reader-${Math.random().toString(36).slice(2)}` + const read = vi.fn().mockResolvedValue({ url: 'https://example.com', content: 'ok' }) + register(providerName, 'https://reader.example.com', () => ({ + name: () => providerName, + search: vi.fn().mockResolvedValue([]), + read, + })) + + await readUrl(' https://example.com ', { provider: providerName, format: 'text', maxTokens: 500 }) + + expect(read).toHaveBeenCalledWith('https://example.com', { format: 'text', maxTokens: 500 }) + }) + + it('throws EmptyUrlError for whitespace-only URLs', async () => { + await expect(readUrl(' ')).rejects.toThrow(EmptyUrlError) + }) + + it('throws ReadNotSupportedError when provider has no read capability', async () => { + const providerName = `search-only-${Math.random().toString(36).slice(2)}` + register(providerName, 'https://search.example.com', () => ({ + name: () => providerName, + search: vi.fn().mockResolvedValue([]), + })) + + await expect(readUrl('https://example.com', { provider: providerName })).rejects.toThrow(ReadNotSupportedError) + }) +}) diff --git a/test/unit/resolve-async.test.ts b/test/unit/resolve-async.test.ts index b5d049c..1951bac 100644 --- a/test/unit/resolve-async.test.ts +++ b/test/unit/resolve-async.test.ts @@ -8,7 +8,7 @@ import { searchAllDetailed } from '../../src/core/all.ts' import { NoProviderAvailableError } from '../../src/core/errors.ts' import '../../src/providers/index.ts' -const envKeys = ['EXA_API_KEY', 'BRAVE_API_KEY', 'TAVILY_API_KEY', 'SERPAPI_API_KEY'] as const +const envKeys = ['EXA_API_KEY', 'BRAVE_API_KEY', 'JINA_API_KEY', 'TAVILY_API_KEY', 'SERPAPI_API_KEY'] as const describe('resolve async', () => { const savedEnv: Record = {} diff --git a/test/unit/resolve.test.ts b/test/unit/resolve.test.ts index 0954aea..912e589 100644 --- a/test/unit/resolve.test.ts +++ b/test/unit/resolve.test.ts @@ -2,7 +2,7 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { detectAvailableProviders, resolveDefaultProvider, listProviders } from '../../src/core/resolve.ts' import '../../src/providers/index.ts' -const envKeys = ['EXA_API_KEY', 'BRAVE_API_KEY', 'TAVILY_API_KEY', 'SERPAPI_API_KEY'] as const +const envKeys = ['EXA_API_KEY', 'BRAVE_API_KEY', 'JINA_API_KEY', 'TAVILY_API_KEY', 'SERPAPI_API_KEY'] as const describe('resolve', () => { const savedEnv: Record = {} @@ -34,9 +34,11 @@ describe('resolve', () => { it('should detect multiple providers', () => { process.env.EXA_API_KEY = 'test-key' process.env.BRAVE_API_KEY = 'test-key' + process.env.JINA_API_KEY = 'test-key' const available = detectAvailableProviders() expect(available).toContain('exa') expect(available).toContain('brave') + expect(available).toContain('jina') }) it('should always include searxng when registered', () => { @@ -48,6 +50,7 @@ describe('resolve', () => { const available = detectAvailableProviders() expect(available).not.toContain('exa') expect(available).not.toContain('brave') + expect(available).not.toContain('jina') expect(available).not.toContain('tavily') expect(available).not.toContain('serpapi') }) @@ -76,6 +79,7 @@ describe('resolve', () => { const names = list.map(p => p.name) expect(names).toContain('brave') expect(names).toContain('exa') + expect(names).toContain('jina') expect(names).toContain('searxng') expect(names).toContain('serpapi') expect(names).toContain('tavily') From fcd4f5f5536dc8fd5b5c64a2adf5dcaeb5235969 Mon Sep 17 00:00:00 2001 From: oritwoen <18102267+oritwoen@users.noreply.github.com> Date: Wed, 20 May 2026 12:49:00 +0200 Subject: [PATCH 2/6] docs: clarify askweb provider scope --- AGENTS.md | 29 ++++++++++++++++++++++------- packages/pi/extensions/askweb.ts | 4 ++-- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 650463f..447ecc7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,16 +6,24 @@ ## OVERVIEW -`askweb` is a unified web search provider for agents and CLI. The goal is to normalize multiple web search backends behind one stable TypeScript API and one stable command-line interface, so agent runtimes do not need provider-specific glue. +`askweb` is a unified web-access provider for agents and CLI. It currently exposes two explicit capabilities: `search` (query → result URLs/snippets) and `read` (URL → normalized page content). Providers are integrations that may implement one or both capabilities; do not force URL readers into `search()`. + +Scope preference from the 2026-05-20 Jina/read session: keep `read` here while it is lightweight, but if read grows into browser rendering, crawling, many read-only providers, or heavy dependencies, split into three packages: search, read, and an umbrella `askweb` aggregator using both. ## STRUCTURE ``` src/ -├── index.ts # Public API barrel and normalized provider catalog -└── cli.ts # citty-based CLI entry point for local and scripted usage -test/ -└── index.test.ts # Basic contract tests for the public API +├── core/ # Registry, shared types/errors, searchAll, readUrl +├── providers/ # Provider adapters; integrations may support search and/or read +├── commands/ # citty CLI subcommands (`search`, `read`, `providers`) +├── index.ts # Public API barrel +├── ai.ts # Vercel AI SDK tools +├── opencode.ts # OpenCode plugin tools +└── cli.ts # CLI entry point +packages/pi/extensions/ +└── askweb.ts # Pi tool/command surface +test/unit/ # Public behavior and provider contract tests .github/workflows/ ├── test.yml # CI: typecheck -> build -> test └── release.yml # npm publish on v* tags @@ -26,7 +34,11 @@ test/ | Task | Location | Notes | |------|----------|-------| | Add public exports | `src/index.ts` | Keep the public surface small and explicit | -| Extend CLI | `src/cli.ts` | Add subcommands with `citty`; keep text and JSON output stable | +| Add/extend providers | `src/providers/` | Keep provider-shaped responses inside the adapter | +| Add search behavior | `src/core/all.ts` + provider adapter | Preserve query → results semantics | +| Add read behavior | `src/core/read.ts` + provider adapter + `src/commands/read.ts` | Preserve URL → content semantics | +| Extend CLI | `src/commands/` + `src/cli.ts` | Add subcommands with `citty`; keep text and JSON output stable | +| Extend agent tools | `src/ai.ts`, `src/opencode.ts`, `packages/pi/extensions/askweb.ts` | Keep names capability-specific (`searchTool`, `readTool`, `askweb_read`) | | Add tests | `test/` | Mirror public behavior, not implementation details | | Change build outputs | `build.config.ts` + `package.json` | Keep `entries` and `exports` aligned | | Change CI flow | `.github/workflows/test.yml` | Order stays `typecheck -> build -> test` | @@ -39,16 +51,19 @@ test/ - Public API stays export-barrel-driven from `src/index.ts` - CLI should be thin and call reusable functions from `src/index.ts` - Prefer normalized models over provider-shaped raw objects +- Keep capability names explicit: `search*` for query → results, `read*`/`readUrl` for URL → content - CLI must support both human-readable and machine-readable JSON output - Keep provider names and capability flags as literal unions where possible -- Default to minimal dependencies; only add HTTP/cache layers when provider adapters land +- Default to minimal dependencies; browser rendering/crawling belongs in a future read package unless explicitly decided otherwise ## ANTI-PATTERNS - Do not leak provider-specific response formats into public API +- Do not hide URL → content behind `SearchProvider.search()` - Do not couple CLI formatting with core data models - Do not add `as any`, `@ts-ignore`, or placeholder unsafe types - Do not introduce CJS compatibility shims +- Do not add browser/runtime-heavy dependencies to the core package without revisiting the read/search split - Do not add network code directly in the CLI - Do not make tests depend on external services diff --git a/packages/pi/extensions/askweb.ts b/packages/pi/extensions/askweb.ts index 4a899d3..4a68232 100644 --- a/packages/pi/extensions/askweb.ts +++ b/packages/pi/extensions/askweb.ts @@ -132,7 +132,7 @@ export default function askwebExtension(pi: ExtensionAPI) { name: "askweb", label: "Askweb Search", description: - "Search the web using one of the configured providers (Brave, Exa, Jina, Tavily, SerpAPI, SearXNG) or fan out to every available provider with provider=all. Always returns {url, title, snippet}; optional fields vary by provider: Exa adds summary/highlights/full text + score/author/image, Jina adds content/text + published date/image/metadata, Tavily adds full raw_content + score, Brave adds extra_snippets, SerpAPI adds thumbnail + position metadata, SearXNG adds engine metadata. Pick provider for the shape you need.", + "Read-only/open-world network search: query one configured provider (Brave, Exa, Jina, Tavily, SerpAPI, SearXNG) or fan out to every available provider with provider=all. Always returns {url, title, snippet}; optional fields vary by provider: Exa adds summary/highlights/full text + score/author/image, Jina adds content/text + published date/image/metadata, Tavily adds full raw_content + score, Brave adds extra_snippets, SerpAPI adds thumbnail + position metadata, SearXNG adds engine metadata. Pick provider for the shape you need.", promptSnippet: "Search the web with askweb. Use provider=all to query every configured provider in parallel.", promptGuidelines: [ @@ -287,7 +287,7 @@ export default function askwebExtension(pi: ExtensionAPI) { name: "askweb_providers", label: "Askweb Providers", description: - "List built-in web search providers and which ones are currently configured via environment variables.", + "Read-only/idempotent local/env status: list built-in web search providers and which ones are currently configured via environment variables.", promptSnippet: "List configured askweb providers.", promptGuidelines: [ "Use askweb_providers before askweb if it is unclear which providers are available.", From 6b9aeb7270fbd01db7530ba0f40cf3e09257200d Mon Sep 17 00:00:00 2001 From: oritwoen <18102267+oritwoen@users.noreply.github.com> Date: Wed, 20 May 2026 13:14:47 +0200 Subject: [PATCH 3/6] fix: address read provider review issues --- README.md | 7 ++-- src/commands/read.ts | 26 +++++++------- src/core/read.ts | 20 +++++++++-- src/core/types.ts | 4 +-- src/index.ts | 4 +-- src/providers/jina.ts | 65 +++++++++++++++++++++++----------- test/unit/ai-tool.test.ts | 17 ++++++++- test/unit/jina.test.ts | 48 ++++++++++++++++++++++--- test/unit/read-command.test.ts | 37 ++++++++++++++++--- test/unit/read.test.ts | 8 ++++- 10 files changed, 183 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 993f2c5..cb6d8a1 100644 --- a/README.md +++ b/README.md @@ -263,8 +263,8 @@ interface ReadResult { html?: string publishedDate?: string image?: string - links?: string[] | Record - images?: string[] | Record + links?: string[] + images?: string[] metadata?: Record } ``` @@ -287,7 +287,8 @@ interface SearchOptions { Read options you can pass to `readUrl`: ```typescript -interface ReadOptions { +interface ReadUrlOptions { + provider?: 'jina' // custom registered provider names are also accepted at runtime format?: 'markdown' | 'text' | 'html' maxTokens?: number targetSelector?: string diff --git a/src/commands/read.ts b/src/commands/read.ts index 0c79b5d..48fb044 100644 --- a/src/commands/read.ts +++ b/src/commands/read.ts @@ -32,7 +32,7 @@ export default defineCommand({ }, }, async run({ args }) { - const { readUrl } = await import('../core/read.ts') + const { readUrl, readProviderNames } = await import('../core/read.ts') const { AuthError, UnknownProviderError, EmptyUrlError, ReadNotSupportedError } = await import('../core/errors.ts') let providerName = args.provider || 'jina' @@ -68,14 +68,14 @@ export default defineCommand({ } if (result.title) { - consola.log(`\x1b[1m\x1b[36m${result.title}\x1b[0m`) + consola.log(`\x1b[1m\x1b[36m${sanitizeTerminalText(result.title)}\x1b[0m`) } - consola.log(` ${result.url}`) + consola.log(` ${sanitizeTerminalText(result.url)}`) if (result.description) { - consola.log(` \x1b[90m${truncateSingleLine(result.description, 160)}\x1b[0m`) + consola.log(` \x1b[90m${truncateSingleLine(sanitizeTerminalText(result.description), 160)}\x1b[0m`) } consola.log('') - consola.log(result.content) + consola.log(sanitizeTerminalText(result.content)) } catch (error) { if (error instanceof EmptyUrlError) { @@ -89,14 +89,8 @@ export default defineCommand({ process.exit(1) } if (error instanceof UnknownProviderError) { - const { providers } = await import('../core/registry.ts') consola.error(`Unknown provider: ${providerName}`) - const available = providers() - if (available.length > 0) { - consola.info(`Available providers: ${available.join(', ')}`) - } else { - consola.info('No providers registered. Import a provider first.') - } + consola.info(`Read-capable providers: ${readProviderNames.join(', ')}`) process.exit(1) } if (error instanceof ReadNotSupportedError) { @@ -144,3 +138,11 @@ function truncateSingleLine(text: string, maxLength: number): string { const singleLine = text.replace(/\s+/g, ' ').trim() return singleLine.length <= maxLength ? singleLine : `${singleLine.slice(0, maxLength - 1)}…` } + +function sanitizeTerminalText(text: string): string { + return text + .replace(/\x1B\][^\x07]*(?:\x07|\x1B\\)/g, '') + .replace(/\x1B\[[0-?]*[ -/]*[@-~]/g, '') + .replace(/\x1B[\x20-\x2F]*[\x30-\x7E]/g, '') + .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, '') +} diff --git a/src/core/read.ts b/src/core/read.ts index f4251b0..4094daa 100644 --- a/src/core/read.ts +++ b/src/core/read.ts @@ -1,12 +1,16 @@ import type { ReadOptions, ReadResult } from './types.ts' +import { builtinProviders } from './providers.ts' import { EmptyUrlError, ReadNotSupportedError } from './errors.ts' import { create } from './registry.ts' +export const readProviderNames = ['jina'] as const +export type ReadProviderName = typeof readProviderNames[number] + export interface ReadUrlOptions extends ReadOptions { - provider?: string + provider?: ReadProviderName | (string & {}) } -const DEFAULT_READ_PROVIDER = 'jina' +const DEFAULT_READ_PROVIDER: ReadProviderName = 'jina' export async function readUrl(url: string, options?: ReadUrlOptions): Promise { const trimmedUrl = url.trim() @@ -16,6 +20,10 @@ export async function readUrl(url: string, options?: ReadUrlOptions): Promise - images?: string[] | Record + links?: string[] + images?: string[] metadata?: Record } diff --git a/src/index.ts b/src/index.ts index 0474069..a501630 100644 --- a/src/index.ts +++ b/src/index.ts @@ -15,8 +15,8 @@ export { register, create, providers, has } from './core/registry.ts' export { searchAll, searchAllDetailed } from './core/all.ts' export type { SearchAllOptions, SearchAllResult, SearchAllResponse, ProviderError } from './core/all.ts' -export { readUrl } from './core/read.ts' -export type { ReadUrlOptions } from './core/read.ts' +export { readProviderNames, readUrl } from './core/read.ts' +export type { ReadProviderName, ReadUrlOptions } from './core/read.ts' export { resolveDefaultProvider, diff --git a/src/providers/jina.ts b/src/providers/jina.ts index b51e094..14cc894 100644 --- a/src/providers/jina.ts +++ b/src/providers/jina.ts @@ -1,7 +1,7 @@ import type { SearchResult, SearchOptions, ReadResult, ReadOptions, SearchProvider, ProviderConfig, ProviderFactory } from '../core/types.ts' import { defaultClient } from '../core/client.ts' import type { Client } from '../core/client.ts' -import { AuthError, normalizeError } from '../core/errors.ts' +import { AuthError, HTTPError, normalizeError } from '../core/errors.ts' import { register } from '../core/registry.ts' interface JinaResult { @@ -20,18 +20,21 @@ interface JinaResult { pageshotUrl?: string } -interface JinaSearchResponse { - code: number - status: number - data?: JinaResult[] | null +interface JinaEnvelope { + code?: number + status?: number + message?: string + error?: string + detail?: unknown meta?: Record } -interface JinaReadResponse { - code: number - status: number +interface JinaSearchResponse extends JinaEnvelope { + data?: JinaResult[] | null +} + +interface JinaReadResponse extends JinaEnvelope { data?: JinaResult | null - meta?: Record } const JINA_MAX_RESULTS = 20 @@ -78,6 +81,7 @@ class JinaProvider implements SearchProvider { 'Accept': 'application/json', } const response = await this.client.getJSON(url, headers) + assertJinaSuccess(response, url) return (response.data ?? []).map(mapSearchResult) } catch (error) { @@ -89,6 +93,7 @@ class JinaProvider implements SearchProvider { try { const requestUrl = `${this.readBaseURL}/${encodeURIComponent(url)}` const response = await this.client.getJSON(requestUrl, readHeaders(this.apiKey, options)) + assertJinaSuccess(response, requestUrl) return mapReadResult(response.data ?? { url, content: '' }) } catch (error) { @@ -98,15 +103,17 @@ class JinaProvider implements SearchProvider { } function deriveReadBaseURL(searchBaseURL: string): string { - return searchBaseURL === 'https://s.jina.ai' ? 'https://r.jina.ai' : searchBaseURL + const match = searchBaseURL.match(/^(https?:\/\/)(.+\.)?s\.jina\.ai$/) + if (!match) return searchBaseURL + return `${match[1]}${match[2] ?? ''}r.jina.ai` } function readHeaders(apiKey: string | undefined, options: ReadOptions | undefined): Record { const headers: Record = { Accept: 'application/json' } if (apiKey) headers.Authorization = `Bearer ${apiKey}` - if (options?.format) headers['X-Respond-With'] = options.format - if (options?.maxTokens !== undefined) headers['X-Max-Tokens'] = String(options.maxTokens) + if (options?.format) headers['X-Return-Format'] = options.format + if (options?.maxTokens !== undefined) headers['X-Token-Budget'] = String(options.maxTokens) if (options?.targetSelector) headers['X-Target-Selector'] = options.targetSelector if (options?.removeSelector) headers['X-Remove-Selector'] = options.removeSelector if (options?.timeout !== undefined) headers['X-Timeout'] = String(options.timeout) @@ -145,8 +152,8 @@ function mapReadResult(result: JinaResult): ReadResult { html: result.html, publishedDate: result.publishedTime, image: firstImage(result.images), - links: result.links ?? undefined, - images: result.images ?? undefined, + links: stringValues(result.links), + images: stringValues(result.images), metadata: resultMetadata(result), } } @@ -156,19 +163,35 @@ function snippetFrom(text: string | undefined): string | undefined { } function firstImage(images: JinaResult['images']): string | undefined { - if (Array.isArray(images)) { - return images.find(isNonEmptyString) - } - if (images && typeof images === 'object') { - return Object.values(images).find(isNonEmptyString) - } - return undefined + return stringValues(images)?.[0] +} + +function stringValues(value: string[] | Record | null | undefined): string[] | undefined { + const values = Array.isArray(value) ? value : value ? Object.values(value) : [] + const strings = values.filter(isNonEmptyString) + return strings.length > 0 ? strings : undefined } function isNonEmptyString(value: unknown): value is string { return typeof value === 'string' && value.length > 0 } +function assertJinaSuccess(response: JinaEnvelope, url: string): void { + const code = response.code + const status = response.status + if ((code !== undefined && code >= 400) || (status !== undefined && status >= 40000)) { + const statusCode = code !== undefined && code >= 400 ? code : Math.floor((status ?? 50000) / 100) + throw new HTTPError(statusCode, url, jinaErrorMessage(response)) + } +} + +function jinaErrorMessage(response: JinaEnvelope): string { + if (response.message) return response.message + if (response.error) return response.error + if (response.detail !== undefined) return JSON.stringify(response.detail) + return `Jina API error: code=${response.code ?? 'unknown'} status=${response.status ?? 'unknown'}` +} + function resultMetadata(result: JinaResult): Record | undefined { const metadata: Record = result.metadata ? { ...result.metadata } : {} diff --git a/test/unit/ai-tool.test.ts b/test/unit/ai-tool.test.ts index e9d9409..7936e11 100644 --- a/test/unit/ai-tool.test.ts +++ b/test/unit/ai-tool.test.ts @@ -254,6 +254,21 @@ describe('searchTool', () => { describe('readTool', () => { beforeEach(() => { mockGetJSON.mockReset() + for (const key of envKeys) { + savedEnv[key] = process.env[key] + delete process.env[key] + } + }) + + afterEach(() => { + for (const key of envKeys) { + if (savedEnv[key] !== undefined) { + process.env[key] = savedEnv[key] + } + else { + delete process.env[key] + } + } }) it('reads a URL with Jina by default', async () => { @@ -275,7 +290,7 @@ describe('readTool', () => { expect(result.content).toBe('Read content') const [url, headers] = mockGetJSON.mock.calls[0] expect(url).toBe('https://r.jina.ai/https%3A%2F%2Fexample.com') - expect(headers).toEqual({ Accept: 'application/json', 'X-Respond-With': 'markdown' }) + expect(headers).toEqual({ Accept: 'application/json', 'X-Return-Format': 'markdown' }) }) it('rejects empty URL', async () => { diff --git a/test/unit/jina.test.ts b/test/unit/jina.test.ts index b920351..aeedac5 100644 --- a/test/unit/jina.test.ts +++ b/test/unit/jina.test.ts @@ -15,7 +15,7 @@ vi.mock('../../src/core/client.ts', () => ({ })) import { create, has } from '../../src/core/registry.ts' -import { AuthError } from '../../src/core/errors.ts' +import { AuthError, HTTPError } from '../../src/core/errors.ts' import type { SearchResult, ReadResult } from '../../src/core/types.ts' // Triggers self-registration of jina provider @@ -153,6 +153,20 @@ describe('jina provider', () => { }) describe('read()', () => { + it('derives regional read hosts from regional search hosts', async () => { + mockGetJSON.mockResolvedValueOnce({ + code: 200, + status: 20000, + data: { url: 'https://example.com/', content: 'Read content' }, + }) + + const provider = create('jina', { baseURL: 'https://eu.s.jina.ai' }) + await provider.read!('https://example.com') + + const [url] = mockGetJSON.mock.calls[0] + expect(url).toBe('https://eu.r.jina.ai/https%3A%2F%2Fexample.com') + }) + it('calls r.jina.ai with encoded URL and JSON accept header without requiring apiKey', async () => { mockGetJSON.mockResolvedValueOnce({ code: 200, @@ -196,8 +210,8 @@ describe('jina provider', () => { expect(headers).toEqual({ Accept: 'application/json', Authorization: 'Bearer test-key', - 'X-Respond-With': 'text', - 'X-Max-Tokens': '500', + 'X-Return-Format': 'text', + 'X-Token-Budget': '500', 'X-Target-Selector': 'main', 'X-Remove-Selector': 'nav', 'X-Timeout': '30', @@ -237,9 +251,35 @@ describe('jina provider', () => { publishedDate: '2024-08-01T00:00:00Z', image: 'https://example.com/hero.png', links: ['https://example.com/a'], - images: { hero: 'https://example.com/hero.png' }, + images: ['https://example.com/hero.png'], metadata: { lang: 'en', warning: 'cached' }, }) }) + it('throws on Jina application-level errors for search responses', async () => { + mockGetJSON.mockResolvedValueOnce({ + code: 401, + status: 40100, + message: 'invalid token', + }) + + const provider = create('jina', { apiKey: 'bad-key' }) + + await expect(provider.search('query')).rejects.toThrow(AuthError) + }) + + it('throws on Jina application-level errors for read responses', async () => { + mockGetJSON.mockResolvedValueOnce({ + code: 422, + status: 42200, + message: 'unsupported url', + }) + + const provider = create('jina', {}) + + await expect(provider.read!('ftp://example.com')).rejects.toMatchObject({ + statusCode: 422, + body: 'unsupported url', + } satisfies Partial) + }) }) }) diff --git a/test/unit/read-command.test.ts b/test/unit/read-command.test.ts index 23de08d..01496e8 100644 --- a/test/unit/read-command.test.ts +++ b/test/unit/read-command.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { EmptyUrlError, ReadNotSupportedError } from '../../src/core/errors.ts' +import { EmptyUrlError, ReadNotSupportedError, UnknownProviderError } from '../../src/core/errors.ts' const mockLog = vi.fn() const mockInfo = vi.fn() @@ -15,6 +15,7 @@ vi.mock('consola', () => ({ })) vi.mock('../../src/core/read.ts', () => ({ + readProviderNames: ['jina'], readUrl: (...args: unknown[]) => mockReadUrl(...args), })) @@ -53,8 +54,9 @@ function runRead(overrides: Partial = {}) { args: makeArgs(overrides), rawArgs: [], cmd: readCommand, - } as ReadRunInput - return readCommand.run!(context) + } satisfies ReadRunInput + if (!readCommand.run) throw new Error('readCommand.run is not defined') + return readCommand.run(context) } describe('read command', () => { @@ -72,9 +74,9 @@ describe('read command', () => { description: 'Example description', content: 'Example content', }) - exitSpy = vi.spyOn(process, 'exit').mockImplementation((() => { + exitSpy = vi.spyOn(process, 'exit').mockImplementation((_code?: string | number | null) => { throw new Error('__EXIT__') - }) as never) + }) writeSpy = vi.spyOn(process.stdout, 'write').mockImplementation(() => true) }) @@ -111,6 +113,22 @@ describe('read command', () => { expect(parsed.content).toBe('Example content') }) + it('strips terminal control sequences from human output', async () => { + mockReadUrl.mockResolvedValueOnce({ + url: 'https://example.com', + title: 'Example \x1B[31mTitle\x1B[0m', + description: 'Description \x1B]0;bad\x07ok', + content: 'Hello \x1B[31mred\x1B[0m \x01world', + }) + + await runRead() + + const contentLine = String(mockLog.mock.calls.at(-1)?.[0]) + expect(contentLine).not.toContain('\x1B') + expect(contentLine).not.toContain('\x01') + expect(contentLine).toBe('Hello red world') + }) + it('exits with a helpful message for empty URL', async () => { await expect(runRead({ url: ' ' })).rejects.toThrow('__EXIT__') @@ -148,4 +166,13 @@ describe('read command', () => { expect(mockError).toHaveBeenCalledWith('Provider does not support read: brave') }) + + it('shows read-capable providers for unknown read providers', async () => { + mockReadUrl.mockRejectedValueOnce(new UnknownProviderError('missing')) + + await expect(runRead({ provider: 'missing' })).rejects.toThrow('__EXIT__') + + expect(mockError).toHaveBeenCalledWith('Unknown provider: missing') + expect(mockInfo).toHaveBeenCalledWith('Read-capable providers: jina') + }) }) diff --git a/test/unit/read.test.ts b/test/unit/read.test.ts index c0ad2f6..58a676c 100644 --- a/test/unit/read.test.ts +++ b/test/unit/read.test.ts @@ -22,7 +22,13 @@ describe('readUrl', () => { await expect(readUrl(' ')).rejects.toThrow(EmptyUrlError) }) - it('throws ReadNotSupportedError when provider has no read capability', async () => { + it('throws ReadNotSupportedError for search-only built-in providers before constructing them', async () => { + delete process.env.EXA_API_KEY + + await expect(readUrl('https://example.com', { provider: 'exa' })).rejects.toThrow(ReadNotSupportedError) + }) + + it('throws ReadNotSupportedError when a custom provider has no read capability', async () => { const providerName = `search-only-${Math.random().toString(36).slice(2)}` register(providerName, 'https://search.example.com', () => ({ name: () => providerName, From 8c0dbd134a53af468e7864e2e1d3ae10a0a2a7b8 Mon Sep 17 00:00:00 2001 From: oritwoen <18102267+oritwoen@users.noreply.github.com> Date: Wed, 20 May 2026 13:33:43 +0200 Subject: [PATCH 4/6] fix: reuse read provider list across tools --- packages/pi/extensions/askweb.ts | 31 ++++++++++++++++--------------- src/ai.ts | 3 +-- src/opencode.ts | 3 +-- 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/packages/pi/extensions/askweb.ts b/packages/pi/extensions/askweb.ts index 4a68232..1edde5c 100644 --- a/packages/pi/extensions/askweb.ts +++ b/packages/pi/extensions/askweb.ts @@ -1,15 +1,17 @@ import type { AgentToolResult, ExtensionAPI } from "@earendil-works/pi-coding-agent" import { Text } from "@earendil-works/pi-tui" import { Type, type Static } from "typebox" -import type { - ProviderError, - ProviderStatus, - ReadOptions, - ReadResult, - SearchAllResult, - SearchOptions, - SearchResult, - WebSearchProviderName, +import { + readProviderNames, + type ProviderError, + type ProviderStatus, + type ReadOptions, + type ReadProviderName, + type ReadResult, + type SearchAllResult, + type SearchOptions, + type SearchResult, + type WebSearchProviderName, } from "askweb" type SearchSingleDetails = { @@ -35,7 +37,7 @@ type SearchDetails = SearchSingleDetails | SearchAllDetails type ReadDetails = { mode: "read" url: string - provider: "jina" + provider: ReadProviderName options: ReadOptions result: ReadResult } @@ -53,8 +55,7 @@ function loadAskweb(): Promise { const PROVIDERS = ["auto", "all", "brave", "exa", "jina", "searxng", "serpapi", "tavily"] as const const PROVIDER_HINT = `Provider to use. One of: ${PROVIDERS.join(", ")}. "auto" (or omit) picks the first available provider from env. Use "all" to query every configured provider in parallel.` -const READ_PROVIDERS = ["jina"] as const -const READ_PROVIDER_HINT = `Read provider to use. One of: ${READ_PROVIDERS.join(", ")}. Defaults to jina.` +const READ_PROVIDER_HINT = `Read provider to use. One of: ${readProviderNames.join(", ")}. Defaults to jina.` const MAX_RESULTS_HARD_CAP = 20 const DEFAULT_MAX_RESULTS = 10 @@ -125,7 +126,7 @@ type SearchParams = Static type ReadParams = Static type EmptyParams = Static type ProviderInput = (typeof PROVIDERS)[number] -type ReadProviderInput = (typeof READ_PROVIDERS)[number] +type ReadProviderInput = (typeof readProviderNames)[number] export default function askwebExtension(pi: ExtensionAPI) { pi.registerTool({ @@ -253,7 +254,7 @@ export default function askwebExtension(pi: ExtensionAPI) { const rawProvider = (params.provider ?? "jina").trim() || "jina" if (!isKnownReadProvider(rawProvider)) { throw new Error( - `Unknown read provider "${rawProvider}". Available: ${READ_PROVIDERS.join(", ")}.`, + `Unknown read provider "${rawProvider}". Available: ${readProviderNames.join(", ")}.`, ) } @@ -402,7 +403,7 @@ function isKnownProvider(name: string): name is ProviderInput { } function isKnownReadProvider(name: string): name is ReadProviderInput { - return READ_PROVIDERS.some((provider) => provider === name) + return readProviderNames.some((provider) => provider === name) } function normalizeReadFormat(format: string | undefined): ReadOptions["format"] { diff --git a/src/ai.ts b/src/ai.ts index 44977ca..c1b40a4 100644 --- a/src/ai.ts +++ b/src/ai.ts @@ -3,13 +3,12 @@ import { z } from 'zod' import { builtinProviders } from './core/providers.ts' import { create } from './core/registry.ts' import { searchAll } from './core/all.ts' -import { readUrl } from './core/read.ts' +import { readProviderNames, readUrl } from './core/read.ts' import { EmptyQueryError, EmptyUrlError } from './core/errors.ts' import { resolveDefaultProvider, listProviders } from './core/resolve.ts' import './providers/index.ts' const providerNames = [...builtinProviders, 'all'] as const -const readProviderNames = ['jina'] as const export const searchTool = tool({ description: 'Search the web using multiple search engines (Brave, Exa, Jina, Tavily, SerpAPI, SearXNG). Returns relevant web pages with titles, URLs, snippets, and optional metadata. Use provider "all" to query all available providers in parallel and get deduplicated results.', diff --git a/src/opencode.ts b/src/opencode.ts index 9f5d9f9..0b64e85 100644 --- a/src/opencode.ts +++ b/src/opencode.ts @@ -4,13 +4,12 @@ import { encode } from '@toon-format/toon' import { builtinProviders } from './core/providers.ts' import { create } from './core/registry.ts' import { searchAll } from './core/all.ts' -import { readUrl } from './core/read.ts' +import { readProviderNames, readUrl } from './core/read.ts' import { resolveDefaultProvider, listProviders } from './core/resolve.ts' import './providers/index.ts' const z = tool.schema const providerNames = [...builtinProviders, 'all'] as const -const readProviderNames = ['jina'] as const const AskwebPlugin: Plugin = async () => ({ tool: { From 7e5f0c6913af8162318d4c59eea164c258ac2161 Mon Sep 17 00:00:00 2001 From: oritwoen <18102267+oritwoen@users.noreply.github.com> Date: Wed, 20 May 2026 13:37:31 +0200 Subject: [PATCH 5/6] docs: keep provider lists single-source --- AGENTS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 447ecc7..0e23e8b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -54,12 +54,14 @@ test/unit/ # Public behavior and provider contract tests - Keep capability names explicit: `search*` for query → results, `read*`/`readUrl` for URL → content - CLI must support both human-readable and machine-readable JSON output - Keep provider names and capability flags as literal unions where possible +- Keep capability provider lists single-source: `src/core/read.ts` exports read-capable names; AI/OpenCode/Pi surfaces import that list instead of mirroring `['jina']` - Default to minimal dependencies; browser rendering/crawling belongs in a future read package unless explicitly decided otherwise ## ANTI-PATTERNS - Do not leak provider-specific response formats into public API - Do not hide URL → content behind `SearchProvider.search()` +- Do not duplicate provider-name arrays across CLI/tool surfaces; update one core export and reuse it - Do not couple CLI formatting with core data models - Do not add `as any`, `@ts-ignore`, or placeholder unsafe types - Do not introduce CJS compatibility shims From 04831efb61c45d9dfe881863b99c6a28146c5606 Mon Sep 17 00:00:00 2001 From: oritwoen <18102267+oritwoen@users.noreply.github.com> Date: Wed, 20 May 2026 13:41:02 +0200 Subject: [PATCH 6/6] fix: keep pi askweb import lazy --- packages/pi/extensions/askweb.ts | 38 ++++++++++++++++---------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/packages/pi/extensions/askweb.ts b/packages/pi/extensions/askweb.ts index 1edde5c..4a41871 100644 --- a/packages/pi/extensions/askweb.ts +++ b/packages/pi/extensions/askweb.ts @@ -1,17 +1,16 @@ import type { AgentToolResult, ExtensionAPI } from "@earendil-works/pi-coding-agent" import { Text } from "@earendil-works/pi-tui" import { Type, type Static } from "typebox" -import { - readProviderNames, - type ProviderError, - type ProviderStatus, - type ReadOptions, - type ReadProviderName, - type ReadResult, - type SearchAllResult, - type SearchOptions, - type SearchResult, - type WebSearchProviderName, +import type { + ProviderError, + ProviderStatus, + ReadOptions, + ReadProviderName, + ReadResult, + SearchAllResult, + SearchOptions, + SearchResult, + WebSearchProviderName, } from "askweb" type SearchSingleDetails = { @@ -55,7 +54,7 @@ function loadAskweb(): Promise { const PROVIDERS = ["auto", "all", "brave", "exa", "jina", "searxng", "serpapi", "tavily"] as const const PROVIDER_HINT = `Provider to use. One of: ${PROVIDERS.join(", ")}. "auto" (or omit) picks the first available provider from env. Use "all" to query every configured provider in parallel.` -const READ_PROVIDER_HINT = `Read provider to use. One of: ${readProviderNames.join(", ")}. Defaults to jina.` +const READ_PROVIDER_HINT = "Read provider to use. Defaults to Jina and is validated against askweb.readProviderNames at execution time." const MAX_RESULTS_HARD_CAP = 20 const DEFAULT_MAX_RESULTS = 10 @@ -126,7 +125,7 @@ type SearchParams = Static type ReadParams = Static type EmptyParams = Static type ProviderInput = (typeof PROVIDERS)[number] -type ReadProviderInput = (typeof readProviderNames)[number] +type ReadProviderInput = ReadProviderName export default function askwebExtension(pi: ExtensionAPI) { pi.registerTool({ @@ -251,10 +250,12 @@ export default function askwebExtension(pi: ExtensionAPI) { throw new Error("URL cannot be empty") } - const rawProvider = (params.provider ?? "jina").trim() || "jina" - if (!isKnownReadProvider(rawProvider)) { + const askweb = await loadAskweb() + const defaultReadProvider: ReadProviderName = askweb.readProviderNames[0] ?? "jina" + const rawProvider = (params.provider ?? defaultReadProvider).trim() || defaultReadProvider + if (!isKnownReadProvider(rawProvider, askweb)) { throw new Error( - `Unknown read provider "${rawProvider}". Available: ${readProviderNames.join(", ")}.`, + `Unknown read provider "${rawProvider}". Available: ${askweb.readProviderNames.join(", ")}.`, ) } @@ -268,7 +269,6 @@ export default function askwebExtension(pi: ExtensionAPI) { noCache: params.noCache, }) - const askweb = await loadAskweb() const result = await askweb.readUrl(url, { provider: rawProvider, ...readOptions }) const header = `[provider=${rawProvider}] read ${result.url}` return { @@ -402,8 +402,8 @@ function isKnownProvider(name: string): name is ProviderInput { return PROVIDERS.some((provider) => provider === name) } -function isKnownReadProvider(name: string): name is ReadProviderInput { - return readProviderNames.some((provider) => provider === name) +function isKnownReadProvider(name: string, askweb: AskwebModule): name is ReadProviderInput { + return askweb.readProviderNames.some((provider) => provider === name) } function normalizeReadFormat(format: string | undefined): ReadOptions["format"] {