diff --git a/docs/api-reference/admin.mdx b/docs/api-reference/admin.mdx index 3e430fa..48af54c 100644 --- a/docs/api-reference/admin.mdx +++ b/docs/api-reference/admin.mdx @@ -1,22 +1,31 @@ --- title: Admin API -description: Manage API keys, gateway configuration, request logs, and provider health via the Ferro Labs AI Gateway admin REST API. All endpoints require a scoped bearer token. -keywords: [AI gateway admin API, gateway management API, provider health API, gateway REST API, admin bearer token] +description: Manage API keys, dashboard sessions, gateway config, request logs, and the audit trail via the Ferro Labs AI Gateway admin REST API under /admin. +keywords: [AI gateway admin API, gateway management API, admin bearer token, dashboard session, audit log API] --- Admin endpoints are mounted under `/admin` and protected with a bearer token. -Authenticate with the **`MASTER_KEY`** (the primary admin credential, generated by -`ferrogw init`) or an admin-scoped API key issued via `POST /admin/keys`. +Authenticate with the **`MASTER_KEY`** (the bootstrap/break-glass admin +credential, generated by `ferrogw init`) or an API key issued via +`POST /admin/keys`. `POST /admin/session` additionally accepts either +credential and exchanges it for a short-lived dashboard session token — the +one admin route that itself needs no prior auth, since the credential it is +given *is* the auth. ```bash -Authorization: Bearer +Authorization: Bearer ``` ## Scopes -- `admin` - full access +- `admin` - full access, including all write endpoints - `read_only` - read endpoints only +Every read endpoint accepts either scope; every write endpoint requires +`admin`. A key request or update naming a scope outside this set is refused +with `400 invalid_scope` before anything is stored — see +[Create a key](#create-a-key--post-adminkeys). + ## Read endpoints - `GET /admin/dashboard` @@ -26,10 +35,15 @@ Authorization: Bearer - `GET /admin/logs` - `GET /admin/logs/stats` - `GET /admin/providers` -- `GET /admin/plugins` — returns `[{ "name": ..., "type": ..., "enabled": ... }]` -- `GET /admin/health` +- `GET /admin/providers/catalog` +- `GET /admin/plugins` — configured plugins: `[{ "name": ..., "type": ..., "enabled": ... }]` +- `GET /admin/plugins/catalog` — plugins this **build ships**, see [Plugin catalog](#plugin-catalog--get-adminpluginscatalog) +- `GET /admin/health` — deep component + provider + MCP health - `GET /admin/config` - `GET /admin/config/history` +- `GET /admin/sessions` — list live dashboard sessions +- `GET /admin/audit` — durable audit trail +- `DELETE /admin/session` — sign out the session that authenticated the request (any scope) ## Write endpoints (admin scope) @@ -43,6 +57,11 @@ Authorization: Bearer - `PUT /admin/config` - `DELETE /admin/config` - `POST /admin/config/rollback/{version}` +- `DELETE /admin/sessions` — sign every operator out +- `DELETE /admin/sessions/{id}` — revoke one session + +`POST /admin/session` is unauthenticated by design (see the intro above) and +is not gated by either scope group. ## Query parameters @@ -58,21 +77,38 @@ Authorization: Bearer - `limit` (default `50`, max `200`) - `offset` (default `0`) -- `stage`, `model`, `provider` +- `stage` — omitted defaults to **terminal stages only** (one row per + request); `all` restores the raw per-plugin-stage stream; any other value + filters to that single stage +- `model`, `provider` +- `api_key_id` — the recorded credential id, or `none` for rows that name no + credential - `since` (RFC3339) `GET /admin/logs/stats`: -- `limit` (top-N buckets, max `100`) +- `limit` (top-N buckets in `by_provider`/`by_model`, max `100`) +- `buckets` (time-series resolution, max `120`) - `stage`, `model`, `provider` - `since` (RFC3339) `DELETE /admin/logs`: - required: `before` (RFC3339) -- optional: `stage`, `model`, `provider` +- optional: `stage`, `model`, `provider` (no `api_key_id` — a purge is scoped + by stage/model/provider only) + +`GET /admin/audit`: + +- `limit` (default `50`, max `200`) +- `offset` (default `0`) +- `action`, `actor_id` +- `outcome` (`ok`, `denied`, or `error`) +- `since` (RFC3339) When request log storage is disabled, log endpoints return `501 not implemented`. +The same applies to session, audit, and config endpoints when their backing +store isn't wired in. ## API key schema @@ -109,18 +145,29 @@ Request: ```json { "name": "ci-pipeline", - "scopes": ["admin"], + "scopes": ["read_only"], "expires_at": "2026-12-31T23:59:59Z" } ``` - `name` is **required**; an empty name returns `400`. -- `scopes` is optional — when omitted it defaults to `["admin"]`. +- `scopes` is optional — when omitted (or `[]`) it defaults to + `["read_only"]`, the least-privilege choice. Send `["admin"]` explicitly to + create an admin-scoped key. +- Any scope outside `admin`/`read_only` returns `400` with code + `invalid_scope`, naming the accepted set. - `expires_at` is optional and must be RFC3339; an invalid value returns `400`. Response (`201 Created`) is the full API key schema with the complete, unmasked `key` shown this one time. +:::tip +Give each operator their own admin-scoped key rather than sharing one — a +shared key can't be revoked for a single person, and every audit row naming it +answers "who did this" with nothing useful. Keep `MASTER_KEY` for bootstrap +and break-glass recovery, not daily sign-in. +::: + ## Read keys — `GET /admin/keys` and `GET /admin/keys/{id}` `GET /admin/keys` returns a JSON array of key objects; `GET /admin/keys/{id}` @@ -142,13 +189,23 @@ Request (all fields optional): ``` Set `clear_expiration: true` to remove an existing expiry (it takes precedence -over `expires_at`). The response is the updated, masked key object. +over `expires_at`). An omitted or empty `scopes` array leaves existing scopes +untouched — it does **not** reset to `read_only`, unlike creation. A scope +outside `admin`/`read_only` returns `400 invalid_scope`. The response is the +updated, masked key object. + +A request that would drop the caller's own admin scope, delete/revoke the +caller's own key, or remove the **last** remaining admin key is refused with +`409` (`self_mutation_forbidden` or `last_admin_key`) rather than locking every +operator out of the admin API. ## Rotate a key — `POST /admin/keys/{id}/rotate` Generates a brand-new `fgw_...` string for the same key id, sets `rotated_at`, and invalidates the previous string immediately. The response is the full key -schema with the **new unmasked `key`** — shown once, like creation. +schema with the **new unmasked `key`** — shown once, like creation. Rotation is +never blocked by the lockout guard above: the response hands back a working +credential, so the caller can't be locked out by rotating their own key. ## Revoke vs delete @@ -161,7 +218,8 @@ schema with the **new unmasked `key`** — shown once, like creation. - `DELETE /admin/keys/{id}` is a **hard** delete: the key record is removed from the store entirely. Response: `204 No Content`. -Both return `404` for an unknown id. +Both return `404` for an unknown id, and both are subject to the same +self-mutation / last-admin-key guard as `PUT`. ## Key usage — `GET /admin/keys/usage` @@ -188,10 +246,68 @@ echoes the applied query parameters. } ``` +## Dashboard sessions + +The embedded web dashboard authenticates with short-lived session tokens +rather than the raw API key, so the credential never sits in browser storage. +Sessions last 24h absolute / 1h idle and carry the scopes of the credential +that minted them. + +### Exchange a credential — `POST /admin/session` + +Unauthenticated route — the presented bearer (an API key or `MASTER_KEY`) is +the authentication: + +```bash +curl -X POST http://localhost:8080/admin/session \ + -H "Authorization: Bearer $MASTER_KEY" +``` + +```json +{ + "token": "fgws_...", + "subject": "ci-pipeline", + "scopes": ["admin"], + "expires_at": "2026-06-18T10:00:00Z" +} +``` + +An invalid or revoked credential returns `401 invalid_api_key` and is recorded +in the [audit trail](#audit-trail--get-adminaudit) as a denied `session.create` +— the highest-value row in the trail for spotting brute-force attempts. + +### Sign out — `DELETE /admin/session` + +Deletes the session row that authenticated *this* request, so the token stops +validating immediately. Available to `read_only` and `admin` alike, since any +signed-in session can sign itself out. `400 not_a_session` if the request was +authenticated with an API key instead of a session token. + +### List and revoke sessions + +- `GET /admin/sessions` — every currently live session: + `{ "data": [ { "id", "credential_id", "subject", "scopes", "created_at", "last_seen_at", "expires_at" } ] }`. + `credential_id` names the key (or `MASTER_KEY`) the session was minted from; + `subject` is the operator-chosen name and isn't unique on its own. +- `DELETE /admin/sessions/{id}` (admin) — revoke one session. Idempotent: an + unknown id still returns `204`, so this can't be used to probe which + session ids exist. +- `DELETE /admin/sessions` (admin) — sign every operator out at once; returns + `{ "revoked": }`. The nuclear option when a session secret may be + compromised — there's no secret to rotate, so this is the equivalent. + ## Request logs — `GET /admin/logs` Returns a `{ data, summary, filters }` envelope where `data` holds request-log -entries: +entries. Each row carries `duration_ms`, `ttft_ms` (streaming only), and +`cost_usd` — all nullable, since a row written before persistence started, or +one the catalog can't price, legitimately carries none of them. + +By default the response is **one row per request** (only the terminal +stage — `after_request` on success, `on_error` on failure — is returned), even +though the logger writes one row per plugin stage internally. Pass +`stage=all` to get the raw per-stage event stream, or `stage=` to filter +to one stage. ```json { @@ -204,19 +320,24 @@ entries: "limit": 50, "offset": 0, "stage": "", + "stages": ["after_request", "on_error"], "model": "", + "api_key_id": "", "provider": "", "since": "" } } ``` +`api_key_id=` narrows to rows served by that credential (matched exactly +against the recorded id, never re-validated against the key store — a deleted +key still has rows). `api_key_id=none` selects rows that name no credential at +all: unauthenticated requests, and rows logged before the column existed. + ## Log statistics — `GET /admin/logs/stats` -Aggregates up to 5000 scanned entries into `by_stage` / `by_provider` / -`by_model` count maps. The `limit` query parameter trims the provider and model -maps to the top-N buckets. `summary.truncated` is `true` when more entries exist -than were scanned. +Aggregates scanned entries into counts, a time series, token/cost totals, and +latency percentiles. ```json { @@ -224,20 +345,48 @@ than were scanned. "total_entries": 5000, "error_entries": 12, "total_tokens": 1284000, - "truncated": true, - "available_entries": 7321, - "scan_limit": 5000 + "prompt_tokens": 812000, + "completion_tokens": 472000, + "cost_usd": 41.27, + "unpriced_requests": 6 }, + "latency_ms": { "p50": 420, "p95": 1380, "p99": 2510, "max": 4102, "mean": 610, "count": 4988 }, + "ttft_ms": { "p50": 180, "p95": 640, "p99": 990, "max": 1500, "mean": 240, "count": 3102 }, "by_stage": { "after_request": 4988, "on_error": 12 }, - "by_provider": { "openai": 4200, "anthropic": 800 }, - "by_model": { "gpt-4o": 3100, "claude-3-5-sonnet": 700 }, - "filters": { "limit": 0, "stage": "", "model": "", "provider": "", "since": "" } + "by_provider": { + "openai": { "count": 4200, "errors": 8, "tokens": 980000, "cost_usd": 32.10, "unpriced": 0 } + }, + "by_model": { + "gpt-4o": { "count": 3100, "errors": 5, "tokens": 720000, "cost_usd": 24.55, "unpriced": 0 } + }, + "top_errors": [ { "message": "upstream_unavailable", "count": 4 } ], + "series": { + "points": [ + { "start": "2026-06-17T10:00:00Z", "requests": 210, "errors": 1, "prompt_tokens": 34000, "completion_tokens": 12000 } + ], + "truncated": false, + "start": "2026-06-17T00:00:00Z", + "end": "2026-06-17T23:00:00Z" + }, + "filters": { "limit": 0, "buckets": 24, "stage": "", "model": "", "provider": "", "since": "" } } ``` +- `by_provider` and `by_model` carry per-group error counts, token totals, and + cost — not bare counts — and `limit` trims each to its top-N by count. +- `latency_ms`/`ttft_ms` are `null` when nothing was measured (a fresh + gateway, or a filter matching only non-streaming rows for `ttft_ms`), not a + misleading `0`. +- `series.truncated` and the `start`/`end` window tell you whether the series + covers the full requested range — a series that stopped short looks + identical to a gateway that went quiet unless this is checked. +- `cost_usd` is a **floor**, not a total: `unpriced_requests` counts requests + whose model the catalog doesn't price, which contribute nothing to the sum. + ## Installed plugins — `GET /admin/plugins` -Returns a JSON array describing the configured plugins: +Returns a JSON array describing the plugins **configured** on this instance, +read live from the running config: ```json [ @@ -246,49 +395,169 @@ Returns a JSON array describing the configured plugins: ] ``` +## Plugin catalog — `GET /admin/plugins/catalog` + +Returns the plugins this **build ships**, independent of what's configured — +fixed for the process lifetime: + +```json +{ + "data": [ + { + "name": "budget", + "type": "ratelimit", + "summary": "Tracks estimated spend per API key and refuses requests once the configured budget is exhausted.", + "settings": ["spend_limit_usd", "input_per_m_tokens", "output_per_m_tokens", "cache_read_per_m_tokens", "cache_write_per_m_tokens", "max_keys", "store_id"], + "fails_open": false + } + ] +} +``` + +`settings` lists only the `config` keys the plugin actually reads — the same +list [`GET /admin/config`](#gateway-configuration) uses to decide which keys +of that plugin's config block to show rather than withhold. `fails_open` +reflects the plugin's `Type()`: logging and metrics plugins fail open, +everything else fails closed (`500`) on an internal plugin error. + +## Health — `GET /admin/health` + +A deeper diagnostic than the unauthenticated [`/readyz`](/operations/monitoring): +bearer-authenticated, so it can safely include MCP failure reasons that +`/readyz` withholds (a server URL, an auth header, a subprocess command line). + +```json +{ + "status": "degraded", + "providers": [ + { "name": "openai", "status": "available", "models": 42 } + ], + "components": [ + { "name": "API", "status": "healthy" }, + { "name": "Key store", "status": "healthy" }, + { "name": "Config store", "status": "healthy" }, + { "name": "Request logs", "status": "healthy" }, + { "name": "Audit log", "status": "unavailable" } + ], + "mcp_servers": [ + { "name": "filesystem", "ready": false, "required": true, "last_error": "stdio transport: exit status 1" } + ] +} +``` + +`status` is `healthy`, `degraded`, or `no_providers`. A component reports +`disabled` when its backing store isn't configured (e.g. no +`REQUEST_LOG_STORE_BACKEND`). The audit store failing **never** downgrades +`status` — it fails open by design, so a broken audit trail doesn't take +traffic-serving out of rotation. An MCP server only downgrades `status` when +it is both `required: true` and not ready; an optional server being down costs +only its own tools. + +## Audit trail — `GET /admin/audit` + +Reads the durable audit trail — key and session lifecycle events, config +mutations, and log purges — written best-effort alongside every consequential +admin action. The write never blocks or fails the action it records; a +persistently unreachable audit store means the structured log line is the +record instead. + +```json +{ + "data": [ + { + "occurred_at": "2026-06-17T10:05:00Z", + "action": "key.create", + "actor": "Ops laptop (key-7f2)", + "actor_id": "master-key:a1b2c3d4", + "target_id": "1f2e3d4c-...", + "outcome": "ok", + "detail": "{\"name\":\"ci-pipeline\",\"scopes\":[\"admin\"]}", + "source_ip": "10.0.0.4", + "trace_id": "..." + } + ], + "summary": { "total_entries": 1, "returned_entries": 1 }, + "filters": { "limit": 50, "offset": 0 } +} +``` + +`outcome` is `ok`, `denied` (a guard refused the action, e.g. `last_admin_key`, +`self_mutation_forbidden`, or an invalid credential at `POST /admin/session`), +or `error` (the action itself failed). `actor` is the display form frozen at +write time (e.g. a key's name); `actor_id` is the credential id alone, for +filtering one operator's history. Nothing here echoes a credential value — +`detail` is redacted before storage, as a backstop, not as license to pass +secrets to it. The store follows the same backend as the key store +(`API_KEY_STORE_BACKEND`); the in-memory default retains only recent entries +and is lost on restart, so a deployment that needs the full history +configures a SQL backend. + ## Gateway configuration The config endpoints operate on the **entire** gateway config object — every -write is a full replace, not a partial patch. Supplying a partial body drops the -omitted fields. +write is a full replace, not a partial patch. Supplying a partial body drops +the omitted fields. -- `GET /admin/config` returns the current full config as JSON. +- `GET /admin/config` returns the current config, scrubbed (see below). - `PUT /admin/config` replaces the running config (`200`, body `{ "status": "updated" }`). - `POST /admin/config` also replaces it (`201`, body `{ "status": "created" }`). - `DELETE /admin/config` resets to the startup config (`{ "status": "deleted" }`). -A rejected config returns `400` (`invalid_config`); a persistence failure returns -`500`. When config management is not wired in, these return `501`. +Both `PUT` and `POST` decode the body **as strictly as a config file loads**: +an unknown key returns `400 invalid_request` naming it, rather than being +silently dropped. A body that decodes but fails schema validation (e.g. an +unroutable `target_key`) returns `400 invalid_config`; a persistence failure +returns `500`. When config management is not wired in, these return `501`. -:::danger -`GET /admin/config` can echo **literal secrets** if they were written inline. -Prefer `${ENV}` references in config values so the rendered config exposes the -variable name, not the credential. +:::warning +`GET /admin/config` **scrubs** the config before serving it — it does not echo +literal secrets. Named string fields (URLs, tokens) have their secret portions +replaced with `[REDACTED]`; a `${VAR}` reference is left as-is, since it names +a value rather than carrying one. **Free-form maps** — `mcp_servers[].env`, +`mcp_servers[].headers`, `observability.exporters[].config`, +`observability.tracing.headers`, and any plugin's `config` keys it doesn't +declare in its [catalog entry](#plugin-catalog--get-adminpluginscatalog) — +are withheld **key and value**, each key replaced by `[REDACTED_KEY_]` +(indexed over the sorted original names, so the response is stable across +calls but names none of them). `aliases` and a plugin's declared settings are +the exception and round-trip normally. + +Because a withheld map's keys are gone, it **can't be edited via `GET` → +edit → `PUT`** — edit those from the config file instead. `PUT`/`POST` also +**refuse** any body containing a `[REDACTED...]` marker anywhere, so +round-tripping an unedited `GET` response back can't silently overwrite a live +credential with placeholder text. ::: ### Config history — `GET /admin/config/history` -Returns the in-memory version history accumulated by admin-API writes since the -process started: - ```json { "data": [ { "version": 1, "updated_at": "2026-06-17T10:05:00Z", - "config": { /* full config snapshot */ }, - "rolled_back_from": null + "config": { /* full config snapshot, scrubbed */ }, + "rolled_back_from": null, + "actor": "master-key:a1b2c3d4" } ], "summary": { "total_versions": 1 } } ``` -:::warning -History is held **in memory only** and is lost on restart. It also covers only -config changes made through the admin API since the current process started — -the file the gateway booted from is not recorded as version 0. +History is **durable whenever a config store is configured** +(`CONFIG_STORE_BACKEND=sqlite|postgres`): it survives a restart, numbers +versions from a single persistent counter, and a rollback can target a +version applied before the current process started. Each entry's `actor` +records the credential that applied it, in the durable case as well as the +fallback. + +:::note +Only a deployment with **no** config store falls back to the in-memory list — +lost on restart, and version numbering restarts from 1 there too. The +in-memory fallback is the exception, not the default: most production +deployments configure a config store. ::: ### Roll back — `POST /admin/config/rollback/{version}` @@ -296,7 +565,8 @@ the file the gateway booted from is not recorded as version 0. Re-applies the config snapshot identified by `{version}` and appends a new history entry whose `rolled_back_from` records the version that was current before the rollback. `{version}` must be a positive integer that exists in -history, otherwise `404`. +history — including a durable version from before this process started — +otherwise `404`. ```json { @@ -306,15 +576,13 @@ history, otherwise `404`. } ``` -## Bootstrap keys (deprecated) +`ADMIN_BOOTSTRAP_KEY`, `ADMIN_BOOTSTRAP_READ_ONLY_KEY`, and `ADMIN_BOOTSTRAP_ENABLED` were removed in v1.4.0 ; `MASTER_KEY` is the only bootstrap credential now. See [Configure the server](/operations/server-settings) for the full env var reference. -The legacy `ADMIN_BOOTSTRAP_KEY` / `ADMIN_BOOTSTRAP_READ_ONLY_KEY` env vars are -**deprecated** and emit deprecation warnings at startup. They are honored only on -first run while the API key store is empty **and** `MASTER_KEY` is unset. Prefer -the `MASTER_KEY` instead. +## Related -```bash -export ADMIN_BOOTSTRAP_KEY=change-me -export ADMIN_BOOTSTRAP_READ_ONLY_KEY=change-me -export ADMIN_BOOTSTRAP_ENABLED=true -``` +- [Authentication](/guides/auth) — `MASTER_KEY`, virtual keys vs. `fgw_...` API keys +- [Request logging](/operations/request-logging) — the plugin that feeds `/admin/logs` +- [Monitoring](/operations/monitoring) — `/health`, `/readyz`, `/metrics` +- [Data handling & security](/security/data-handling) — credential redaction across the gateway +- [API errors](/api-reference/errors) — the shared error envelope +- [Dashboard](/guides/dashboard) — the UI built on this API diff --git a/docs/api-reference/endpoints.mdx b/docs/api-reference/endpoints.mdx index c53286e..1cf159c 100644 --- a/docs/api-reference/endpoints.mdx +++ b/docs/api-reference/endpoints.mdx @@ -1,26 +1,47 @@ --- title: Gateway endpoints -description: All HTTP endpoints exposed by the Ferro Labs AI Gateway — health checks, Prometheus metrics, OpenAI-compatible inference endpoints, model listing, and admin routes. -keywords: [AI gateway endpoints, LLM proxy endpoints, OpenAI compatible endpoints, gateway HTTP API, Prometheus metrics endpoint] +description: "Reference for every gateway HTTP endpoint: health and readiness probes, OpenAI-compatible routes, model listing, and the governed /v1/* pass-through proxy." +keywords: [AI gateway endpoints, LLM proxy endpoints, OpenAI compatible endpoints, gateway HTTP API, Prometheus metrics endpoint, pass-through proxy] --- -## Core endpoints +The gateway exposes three kinds of routes: unauthenticated orchestrator +probes, natively-handled OpenAI-compatible inference endpoints, and a +transparent `/v1/*` pass-through proxy for everything else. Every +natively-handled route accepts exactly one method (plus `HEAD` on a `GET` +route) — a wrong method returns **405** with an `Allow` header naming what the +route supports, never a silent fall-through to the pass-through proxy below +it. See [Errors](/api-reference/errors) for the full status-code taxonomy. -- `GET /health` - provider availability and model counts -- `GET /metrics` - Prometheus metrics -- `GET /dashboard` - minimal admin UI -- `GET /v1/models` - aggregated model list +## Health, readiness, and metrics -## Model listing — `GET /v1/models` +| Endpoint | Method | Auth | Description | +|---|---|---|---| +| `/health` | GET | none | Per-provider status, circuit state, and model count. Returns `503` with `status: "no_providers"` when no provider is registered. | +| `/livez` | GET | none | Liveness — the process is up. Performs no dependency checks and always returns `200`. | +| `/readyz` | GET | none | Readiness — the gateway can serve traffic. `200` when at least one configured target is routable; `503` with `reason: "no routable targets"` otherwise. The body lists every configured target's `routable` state and, if MCP servers are configured, their readiness. Answers are cached for 1 second so a probe burst costs one evaluation, not one per caller. | +| `/metrics` | GET | `read_only` or `admin` scope | Prometheus metrics. | + +A target's provider can be registered (its credential env var is set) without +being **routable** — routability also requires the target to appear in +`targets[]` and its circuit breaker to be closed. `/health` and `/readyz` +report circuit state per provider name; `/readyz` additionally reports it per +configured target. See [Server settings](/operations/server-settings) for the +readiness contract in full. + +## Model and capability discovery + +### `GET /v1/models` Returns the standard OpenAI envelope `{ "object": "list", "data": [...] }`. Each -entry starts from the minimal OpenAI model shape (`id`, `object`, `owned_by`) and -is enriched from the gateway model catalog. The extra fields are `omitempty`, so -models without a catalog entry return only the base three and clients that read -just `id`/`object`/`owned_by` keep working. +entry starts from the minimal OpenAI model shape (`id`, `object`, `owned_by`, +`created`) and is enriched from the gateway's model catalog when a catalog +entry exists. The catalog fields are `omitempty`, so a model with no catalog +entry returns only the base shape and clients that read just +`id`/`object`/`owned_by` keep working. | Field | Type | Meaning | |---|---|---| +| `created` | integer | Unix timestamp from live provider discovery; `0` when the model came from the catalog or an operator declaration instead | | `mode` | string | Model class, e.g. `chat`, `embedding`, `image` | | `context_window` | integer | Maximum input context in tokens | | `max_output_tokens` | integer | Maximum tokens the model can emit | @@ -41,6 +62,7 @@ just `id`/`object`/`owned_by` keep working. "id": "gpt-4o", "object": "model", "owned_by": "openai", + "created": 0, "mode": "chat", "context_window": 128000, "max_output_tokens": 16384, @@ -52,52 +74,166 @@ just `id`/`object`/`owned_by` keep working. } ``` -## OpenAI compatible endpoints +The list is **one entry per model id**, owned by the first configured target +that serves it (target order, not registration order). A model two targets +both serve is listed once — the OpenAI `/v1/models` contract is keyed by +`id`, so listing it twice would just mean a client's id-keyed map silently +keeps whichever entry came last. The listing includes models the catalog and +live discovery know about *and* any model an operator declared under +`targets[].models` in config, but excludes anything the active routing +strategy would refuse outright (for example, `cost-optimized` with +`unpriced_strategy: skip` omits models the catalog has no price for). See +[Configuration](/getting-started/configuration) for the `targets[].models` +contract. -- `POST /v1/chat/completions` -- `POST /v1/completions` (legacy) -- `POST /v1/embeddings` -- `POST /v1/images/generations` +### `GET /v1/capabilities` -## Proxy pass-through +Returns, per provider, which OpenAI chat parameters that provider forwards, +translates, or cannot express — plus, for providers with a restricted +`response_format` on image generation, the formats they accept. -Any `/v1/*` request that the gateway does not handle natively is transparently -reverse-proxied to the selected provider. These paths work **only** for -providers that implement the proxiable-provider contract (they expose an upstream -base URL and auth headers); for non-proxiable providers the request returns -`501`. Commonly proxied paths include: - -- `/v1/files` -- `/v1/batches` -- `/v1/fine_tuning` -- `/v1/responses` -- `/v1/audio/*` -- `/v1/images/edits` -- `/v1/realtime` - -## Provider selection +```json +{ + "providers": { + "openai": { "temperature": "forward", "logit_bias": "forward" }, + "anthropic": { "temperature": "forward", "logit_bias": "unsupported" } + }, + "image_response_formats": { + "gemini": ["url"] + } +} +``` -For proxy routes, the gateway resolves the provider in this order: +The providers listed are exactly the ones `/v1/models` lists models for — the +set a configured `targets[]` entry actually routes to — so a parameter marked +`forward` here is never `404`'d by a routed surface a moment later. See +[Providers](/providers) for the full per-provider parameter matrix rendered +as a table. -1. `X-Provider` header (for example: `openai` or `anthropic`) -2. top-level `model` field in the JSON body (peeked without consuming the body) +## OpenAI-compatible endpoints -If neither resolves a provider, the gateway returns **`400`** -(`provider_not_resolved`). If a provider resolves but does not support -pass-through, it returns **`501`** (`proxy_not_supported`). +These routes are handled natively — request and response bodies are +translated to and from each provider's own wire format, and every one of +them goes through the shared routing pipeline (retry, circuit breaker, +per-target concurrency, plugins, request logging). -## Proxy request and response contract +| Endpoint | Method | Notes | +|---|---|---| +| `/v1/chat/completions` | POST | Supports `stream: true`. | +| `/v1/completions` | POST | Legacy — wrapped as a single-message chat completion and routed identically; no direct pass-through to a provider's own completions route exists. | +| `/v1/embeddings` | POST | | +| `/v1/images/generations` | POST | | +| `/v1/audio/speech` | POST | Text-to-speech; JSON in, raw audio bytes out. | +| `/v1/audio/transcriptions`, `/v1/audio/translations` | POST | Multipart file upload, capped at 25 MiB. | +| `/v1/rerank` | POST | Cohere-v2 request/response contract. | +| `/v1/moderations` | POST | OpenAI contract. | + +Not every provider implements every surface — rerank and moderations are +each supported by only a handful of the 30 providers. See +[Providers](/providers) for the endpoint-support matrix. + +## Files and batches (`/v1/files`, `/v1/batches`) + +`/v1/files*` and `/v1/batches*` are a transparent pass-through to a single +configured backend, `batch_target` — a `targets[].virtual_key` naming a +provider with a batch-capable OpenAI-compatible surface (`openai`, +`azure-openai`, `groq`, `novita`, `qwen`). Unlike every routed surface, these +carry no `model`: a batch job references an uploaded `input_file_id`, and a +bare `GET /v1/files/{id}` is an opaque provider-scoped id with no routing +hint. So the gateway's model-based routing does not apply, ids are forwarded +**native** (never rewritten), and every method the two APIs use (`GET`, +`POST`, `DELETE`) is forwarded — there is no per-method `405` guard on this +surface, because it *is* the pass-through to that backend. + +`batch_target` is optional. **When it is unset, or names a target whose +provider is not batch-capable, every route under `/v1/files` and +`/v1/batches` answers `501`.** + +## Responses (`/v1/responses`) + +`POST /v1/responses` routes **like chat**: it carries a `model` field, is +resolved through the routing index, and runs the full governed pipeline +(plugins, guardrails, circuit breaker, per-target concurrency, request log). +Unlike the generic `/v1/*` pass-through below, it is also **priced** — the +Responses API returns a `usage` object (on the JSON body, or on the terminal +SSE event) that the gateway tees out as the response streams through, +without altering a byte, so cost accounting and the request log's `cost_usd` +column are populated rather than left unknown. `openai` and `xai` serve the +OpenAI Responses contract byte-compatibly. + +The stateful **id sub-routes** — `GET`/`DELETE /v1/responses/{id}`, +`POST /v1/responses/{id}/cancel`, `GET /v1/responses/{id}/input_items` — +carry no model and reference an opaque, provider-scoped id, so they always +pin to a single configured `responses_target` (the same native-id, zero-state +pattern as batch). **They answer `501` when `responses_target` is unset**; +`POST /v1/responses` (create) is unaffected and still routes by model. A +provider that is not OpenAI-wire (see below) is refused `501` on this whole +surface. + +## Proxy pass-through (`/v1/*`) + +Any `/v1/*` request the gateway does not handle natively — `/v1/fine_tuning`, +`/v1/images/edits`, `/v1/vector_stores`, `/v1/realtime`, and any other OpenAI +resource path — is transparently reverse-proxied to a provider. This is +**not** a bypass: a pass-through request runs the same governance the routed +surfaces do — `before_request`/`after_request`/`on_error` plugins (tagged +`surface: "passthrough"`), the target's circuit breaker, its per-target +concurrency limiter, `request_timeout` (when configured), and request +logging. It differs from a routed request in two ways: there is **no automatic retry** (the request body has already been +streamed upstream by the time a failure is known, and most of these endpoints +are not idempotent — a retried `/v1/files` upload is a second file), and cost +is recorded as **unpriced** rather than a known zero, since the response body +is opaque by construction. + +### Provider resolution + +The gateway resolves a target in this order: + +1. `X-Provider` request header (for example `X-Provider: openai`) +2. the top-level `model` field in the JSON body, resolved through the same + routing index `/v1/models` and the routed surfaces use — not a scan of + each provider's advisory `SupportsModel` + +| Condition | Response | +|---|---| +| `X-Provider` names a provider no configured target serves | `404 provider_not_found` | +| Body names a `model` no configured target owns | `404 model_not_found` — the same answer the natively-handled surfaces give for an unroutable model | +| Neither `X-Provider` nor a `model` field is present | `400 provider_not_resolved` | +| The resolved provider does not implement the pass-through contract, or is a native (non-OpenAI-wire) provider | `501 proxy_not_supported` | +| The request path contains a traversal segment (`..`, encoded or repeated) | `400 invalid_proxy_path`, refused before any credential is attached | + +A model owned by no configured target is **never forwarded** — the gateway +does not guess. Eight providers are native-wire and always refuse the +pass-through with `501`, serving the same functionality only through their +translated native endpoints instead: `anthropic`, `azure-foundry`, +`azure-openai`, `bedrock`, `cohere`, `gemini`, `replicate`, `vertex-ai`. +`ollama-cloud` is the one provider with no pass-through support at all (it +exposes no proxiable base URL or auth headers). + +### Request and response contract When a request is proxied, the gateway rewrites it before forwarding: -- The inbound client `Authorization` header is **stripped** and replaced with the - resolved provider's own authentication headers (the gateway injects upstream - credentials — clients never send the provider key directly). +- The inbound client `Authorization` header is **stripped** and replaced with + the resolved provider's own authentication headers — clients never send the + provider credential directly. - The `X-Provider` header is removed before the request leaves the gateway. - Standard `X-Forwarded-*` headers are set. -On the response, the gateway adds: +On the response, the gateway adds `X-Gateway-Provider` (the name of the +provider that served the request) and scans non-2xx bodies to redact any +echoed credential before it reaches the client. + +An upstream connection failure surfaces as `502 upstream_error`. A +before-request content guardrail configured on the deployment that cannot +read the request body (a multipart upload, binary audio, or anything not +JSON) refuses the request with `400` rather than forwarding it uninspected. -- `X-Gateway-Provider` — the name of the provider that served the request. +## Related -An upstream connection failure surfaces as **`502`** (`proxy error: ...`). +- [API overview](/api-reference/overview) +- [Errors](/api-reference/errors) +- [Streaming](/api-reference/streaming) +- [Admin API](/api-reference/admin) +- [Providers](/providers) +- [Server settings](/operations/server-settings) diff --git a/docs/api-reference/errors.mdx b/docs/api-reference/errors.mdx index e91209d..8d9f0f8 100644 --- a/docs/api-reference/errors.mdx +++ b/docs/api-reference/errors.mdx @@ -1,10 +1,10 @@ --- title: Error reference -description: HTTP error reference for the Ferro Labs AI Gateway — OpenAI-style error envelope plus a status-code table marking each error as Gateway- or Provider-sourced. -keywords: [AI gateway errors, error codes, OpenAI error format, proxy error source, rate_limit_exceeded, 429, 502 upstream, stream_error] +description: HTTP status codes the Ferro Labs AI Gateway's OpenAI-style error envelope returns, 400 to 504, with Retry-After rules and gateway vs. provider error sources. +keywords: [AI gateway errors, error codes, OpenAI error format, insufficient_quota, model_not_found, provider_saturated, upstream_unavailable, Retry-After, 402, 404, 429, 503, stream_error] --- -The Ferro Labs AI Gateway returns errors in the same JSON shape as the OpenAI API, so existing OpenAI SDK error handling works unchanged. Because the gateway is a proxy in front of 30 upstream providers, every error has a **source**: it either originates in the gateway itself (bad request, auth, rate limit, plugin rejection) or is passed through from the upstream provider. The table below labels each row so you can tell the two apart — the column that pure proxy users need most. +The Ferro Labs AI Gateway returns errors in the same JSON shape as the OpenAI API, so existing OpenAI SDK error handling works unchanged. Because the gateway is a proxy in front of 30 upstream providers, every error has a **source**: it either originates in the gateway itself (bad request, auth, budget, plugin rejection) or is passed through from the upstream provider. The table below labels each row so you can tell the two apart — the column that pure proxy users need most. ## Error envelope @@ -13,9 +13,9 @@ Every gateway-generated error is a single JSON object with one `error` field. Th ```json { "error": { - "message": "rate limit exceeded", + "message": "the gateway is at capacity for this request; retry shortly", "type": "rate_limit_error", - "code": "rate_limit_exceeded" + "code": "provider_saturated" } } ``` @@ -23,42 +23,60 @@ Every gateway-generated error is a single JSON object with one `error` field. Th | Field | Description | | --- | --- | | `message` | Human-readable explanation of what went wrong. | -| `type` | Broad error category (`invalid_request_error`, `authentication_error`, `permission_error`, `not_found_error`, `rate_limit_error`, `upstream_error`, `server_error`). | -| `code` | Stable, machine-readable identifier you can branch on (e.g. `model_not_found`, `invalid_api_key`, `rate_limit_exceeded`). | +| `type` | Broad error category (`invalid_request_error`, `authentication_error`, `permission_error`, `not_found_error`, `rate_limit_error`, `insufficient_quota`, `upstream_error`, `server_error`). | +| `code` | Stable, machine-readable identifier you can branch on (e.g. `model_not_found`, `insufficient_quota`, `provider_saturated`). | :::note -Errors the gateway generates carry `Content-Type: application/json` and this envelope. Two exceptions are **not** wrapped in the envelope: an upstream **connection failure** on a pass-through `/v1/*` route returns a plain-text `502 Bad Gateway` body (`proxy error: `), and any response forwarded verbatim by the pass-through proxy keeps the upstream provider's own status code and body. +Errors the gateway generates — including a pass-through connection failure, now reported as `502 upstream_error` in this same envelope — carry `Content-Type: application/json`. Two things sit genuinely outside it: a response the pass-through proxy forwards **verbatim** keeps the upstream provider's own status code and body (whatever shape that provider uses), and a path this instance doesn't route at all falls through to the embedded dashboard, answering a plain-text `404 page not found` unless the request is a browser asking for HTML. See the notes under [Status codes](#status-codes). ::: ## Status codes The **Source** column tells you where the error was decided: -- **Gateway** — the gateway rejected or could not route the request; the upstream provider was never contacted (or the gateway shaped the failure itself). -- **Provider** — the gateway forwarded the upstream provider's status and body unchanged (pass-through `/v1/*` routes return the provider's exact response). +- **Gateway** — the gateway rejected, could not route, or refused the request itself; either no provider was contacted, or the gateway re-shaped a provider's status into its own error taxonomy. +- **Provider** — the failure is the upstream provider's: forwarded verbatim by the pass-through proxy, or reported through the gateway's envelope after it re-attributed a provider status/failure to the caller. + +:::tip Retry-After +`Retry-After` is set in exactly two cases, and it can win on a status other than `429`: whenever the underlying error carries an upstream wait hint (`Retry-After` or `X-RateLimit-Reset` from the provider's own response) it is forwarded **whatever the final status ends up being** — a `502 upstream_error` built from a throttled `503` still keeps that wait. Otherwise, if the final status is `429`, the gateway adds a constant `Retry-After: 1`. No other status gets the header — `402 insufficient_quota` in particular never does, so a client's first refusal is its last request rather than a retry loop against an answer that cannot change. +::: | HTTP Status | `type` / `code` | Meaning | Source | What to do | | --- | --- | --- | --- | --- | -| `400` | `invalid_request_error` / `invalid_request` | Malformed JSON body or failed request validation on `/v1/chat/completions`. | **Gateway** | Fix the request body; ensure required fields like `model` and `messages` are present and valid. | -| `400` | `invalid_request_error` / `model_not_found` | No registered provider serves the requested `model`. | **Gateway** | Use a model offered by a configured provider; check `GET /v1/models`. | -| `400` | `invalid_request_error` / `streaming_not_supported` | The resolved provider for the model cannot stream. | **Gateway** | Retry with `stream: false`, or route the model to a streaming-capable provider. | -| `400` | `invalid_request_error` / `provider_not_resolved` | Pass-through `/v1/*` request with no provider resolvable. | **Gateway** | Set the `X-Provider` header (e.g. `X-Provider: openai`) or include a top-level `model` field in the body. | -| `400` | `invalid_request_error` / `request_rejected` | A `before_request` plugin (e.g. word filter, max-token) rejected the request. | **Gateway** | Adjust the request to satisfy the guardrail, or change the plugin config. | +| `400` | `invalid_request_error` / `invalid_request` | Malformed JSON body, or a required field failed validation (e.g. missing `model`). | **Gateway** | Fix the request body. | +| `400` | `invalid_request_error` / `request_rejected` | A `before_request` plugin (word filter, max-token, a custom guardrail) rejected the request. | **Gateway** | Adjust the request to satisfy the guardrail, or change the plugin config. `message` carries the plugin's own reason. | +| `400` | `invalid_request_error` / `unsupported_parameter` | The request used an OpenAI parameter the resolved provider can't express, and `compatibility.on_unsupported_param` is set to `reject`. | **Gateway** | Drop the named parameter, or route the model to a provider that supports it — check `GET /v1/capabilities`. | +| `400` | `invalid_request_error` / `provider_not_resolved` | Pass-through `/v1/*` request named no provider and no target owns any `model` in the body. | **Gateway** | Set the `X-Provider` header (e.g. `X-Provider: openai`) or include a top-level `model` field in the body. | +| `400` | `invalid_request_error` / `invalid_proxy_path` | The pass-through path contains a `..`-style traversal segment (checked before any provider credential is attached). | **Gateway** | Remove the traversal segment from the request path. | +| `400` | `invalid_request_error` / `streaming_not_supported` | `POST /v1/completions` (the legacy endpoint) was called with `stream: true` — that endpoint never streams, regardless of provider. | **Gateway** | Use `/v1/chat/completions` for streaming, or drop `stream` on `/v1/completions`. | | `401` | `authentication_error` / `missing_api_key` | Missing or non-`Bearer` `Authorization` header on a protected route. | **Gateway** | Send `Authorization: Bearer `. | | `401` | `authentication_error` / `invalid_api_key` | The bearer token is unknown or revoked. | **Gateway** | Issue or rotate a valid key. | | `401` | `authentication_error` / `authentication_required` | A scope-checked route had no authenticated key in context. | **Gateway** | Authenticate before calling the endpoint. | -| `403` | `permission_error` / `insufficient_scope` | Authenticated key lacks the required scope (e.g. `read_only` calling an admin write). | **Gateway** | Use a key with the `admin` scope, or request the needed scope. | -| `404` | `not_found_error` / `not_found_error` | Unknown route or missing resource. | **Gateway** | Check the path and HTTP method. | -| `429` | `rate_limit_error` / `rate_limit_exceeded` | Per-IP token bucket exhausted, **or** a rate-limit plugin rejected, **or** the budget plugin's per-key USD limit was hit. | **Gateway** | Back off and retry. No `Retry-After` header is sent — use client-side exponential backoff. | -| `500` | `server_error` / `routing_error` | Generic routing failure or an unclassified error from the routing layer (including circuit-open and upstream errors surfaced on native endpoints). | **Gateway** | Retry; if persistent, inspect gateway logs and provider health. | -| `500` | `server_error` / `internal_error` | Internal configuration fault (e.g. a provider with an unparseable base URL on a proxy route). | **Gateway** | Fix the provider configuration. | -| `501` | `invalid_request_error` / `proxy_not_supported` | The resolved provider does not implement proxy pass-through for this `/v1/*` endpoint. | **Gateway** | Use a provider that supports pass-through, or call a natively-handled endpoint. | -| `502` | `upstream_error` / `response_rejected` | An `after_request` plugin rejected the upstream response. | **Gateway** | Loosen the guardrail or fix what the provider returned. | -| `502` | *(plain text)* `proxy error: ` | Upstream connection/transport failure while proxying a `/v1/*` request. | **Provider** | Transient — retry. If it persists, check provider availability and network egress. | -| `503` | *(forwarded verbatim)* | Upstream provider returned `503 Service Unavailable`; the pass-through proxy forwards its status and body unchanged. | **Provider** | Retry with backoff; the provider is overloaded or in maintenance. | - -:::tip -On pass-through `/v1/*` routes the gateway is transparent: any `4xx`/`5xx` the upstream returns (including `429`, `500`, `502`, `503`) is forwarded with the **provider's own** status code and body. Treat those as Provider-sourced even though they reach you through the gateway. +| `402` | `insufficient_quota` / `insufficient_quota` | The `budget` plugin's per-key USD cap is exhausted. | **Gateway** | Stop retrying — this status is deliberately outside every OpenAI SDK's retry set and carries no `Retry-After`. Wait for cost roll-off, or raise/reset the key's budget. | +| `403` | `permission_error` / `insufficient_scope` | Authenticated key lacks the required scope (e.g. `read_only` calling an admin write, or a non-`admin` key hitting `/debug/*`). | **Gateway** | Use a key with the `admin` scope, or request the needed scope. | +| `404` | `invalid_request_error` / `model_not_found` | No configured target serves the requested model. Two distinct causes share this exact status/type/code: no target names a provider for it at all (routing never called anyone), **or** the provider that owns it has since retired the model upstream (routing called the right target and it said no). | **Gateway** or **Provider** | Use a model a configured target serves — check `GET /v1/models`. This also covers a target that can't stream, embed, or generate images for the model; there is no separate status for a capability miss. | +| `404` | `invalid_request_error` / `provider_not_found` | Pass-through `/v1/*` request set `X-Provider` to a name no configured target serves. | **Gateway** | Use a provider name from `GET /v1/models`, or omit `X-Provider` and let the body's `model` resolve it. | +| `404` | `not_found_error` / `resource_not_found` | An admin resource named by id — an API key, a config history version — doesn't exist. | **Gateway** | Check the id; list the resource first (`GET /admin/keys`, `GET /admin/config/history`). | +| `404` | *(plain text)* `404 page not found` | The path matches no route this instance serves (not `/v1/*`, `/admin/*`, a probe, or a dashboard asset), and the request either isn't `GET`/`HEAD` or didn't ask for `text/html`. | **Gateway** | Check the path and HTTP method — this is Go's default 404, not the JSON envelope. The same unmatched path answers a browser `GET` with the dashboard's `200` app shell instead. | +| `405` | `invalid_request_error` / `method_not_supported` | Wrong HTTP method on a route the gateway itself handles. | **Gateway** | Use a method from the response's `Allow` header (every `GET` route also accepts `HEAD`). | +| `429` | `rate_limit_error` / `rate_limit_exceeded` | The per-IP token bucket is exhausted, a `rate-limit` plugin rejected the request, **or** the resolved provider itself returned `429` on a routed surface (its status and code are kept as-is). | **Gateway** or **Provider** | Back off and retry — `Retry-After` is always set (see the tip above). | +| `429` | `rate_limit_error` / `provider_saturated` | The target's `concurrency.max_concurrency` + `queue_size` are both full; this request would exceed both. | **Gateway** | Back off and retry (`Retry-After: 1`), or raise `targets[].concurrency`, or add another target. | +| `500` | `server_error` / `plugin_error` | A plugin's `Execute` returned an error — it broke rather than denying the request with `Reject`. Guardrail, auth, ratelimit, and transform plugins fail **closed** on this; logging/metrics plugins fail open and never reach the client. | **Gateway** | Retry; if persistent, check the plugin's own logs/dependency (e.g. a rate-limit plugin's backend store being unreachable). | +| `500` | `server_error` / `internal_error` | Internal configuration fault — e.g. an unparseable provider base URL discovered while building a pass-through request (`/v1/*`, `/v1/files*`, `/v1/batches*`). | **Gateway** | Fix the provider's `_BASE_URL` configuration. | +| `500` | `server_error` / `routing_error` | Any routing or plugin failure not covered by a more specific status above. Rare — most failure modes now have their own classification. | **Gateway** | Retry; if persistent, inspect gateway logs. | +| `501` | `invalid_request_error` / `proxy_not_supported` | The resolved provider can't serve this as an OpenAI-compatible pass-through — a native-wire-only provider (Anthropic, Gemini, Bedrock, Cohere, Vertex AI, Azure) or one with no proxy support at all. | **Gateway** | Use the provider's natively-handled endpoint (chat, embeddings, images) instead of an unhandled `/v1/*` path. | +| `501` | `invalid_request_error` / `batch_not_configured` | `batch_target` is unset, or names a target whose provider isn't registered or isn't batch-capable. | **Gateway** | Set `batch_target` to a configured target on a batch-capable provider (`openai`, `azure-openai`, `groq`, `novita`, `qwen`). | +| `501` | `invalid_request_error` / `responses_not_configured` | `responses_target` is unset, or its provider isn't registered — applies only to the stateful `/v1/responses/{id}` sub-routes (retrieve/cancel/delete/input\_items). `POST /v1/responses` (create) still routes by model and is unaffected. | **Gateway** | Set `responses_target` to a configured target. | +| `502` | `upstream_error` / `response_rejected` | An `after_request` plugin rejected the upstream response. | **Gateway** | Loosen the guardrail, or fix what the provider returned. | +| `502` | `upstream_error` / `upstream_auth_error` | The resolved provider rejected the gateway's own credential (upstream `401`/`403`). | **Provider** | This is an operator-side credential problem, not the caller's — rotate the provider API key in the gateway's environment. | +| `502` | `upstream_error` / `upstream_error` | Either the provider returned an unclassified `5xx` on a routed surface, or the gateway couldn't reach or read the provider at all while proxying `/v1/*`, `/v1/files*`, `/v1/batches*`, or `/v1/responses*`. | **Provider** | Transient — retry with backoff. If persistent, check the provider's status page and the gateway's outbound network egress. | +| `503` | `upstream_error` / `upstream_unavailable` | The target's circuit breaker is open, so the request was refused **without** an upstream call this time. | **Gateway** | Retry shortly. Check `gateway_circuit_breaker_state{provider=""}` and `GET /admin/logs?provider=` for what tripped it. | +| `503` | *(forwarded verbatim)* | The provider returned `503 Service Unavailable` on the raw, unhandled `/v1/*` pass-through; its status and body are forwarded exactly as received. | **Provider** | Retry with backoff; the provider is overloaded or in maintenance. | +| `504` | `upstream_error` / `upstream_timeout` | The resolved provider's own connection timed out. | **Provider** | Transient — retry with backoff. | +| `504` | `upstream_error` / `gateway_timeout` | The gateway's own `request_timeout` elapsed, or the caller's own context deadline was exceeded, before any provider produced a usable response. | **Gateway** | Raise `request_timeout` if the model genuinely needs longer, or check whether the target is slow or unhealthy. | + +:::tip Pass-through is transparent, connection failures are not +On the raw `/v1/*` pass-through (any endpoint the gateway doesn't handle natively — `/v1/fine_tuning`, `/v1/realtime`, and similar) a response the upstream actually sent — any `2xx`–`5xx`, `429` and `503` included — is forwarded with the **provider's own** status code and body, credentials redacted. A failure to reach or read the upstream at all (DNS, TLS, connection refused, a truncated body) is different: nothing came back to forward, so the gateway reports it itself as `502 upstream_error` in the standard JSON envelope described above. ::: ## Streaming errors @@ -68,7 +86,7 @@ Once a streaming response has started, the HTTP status line is already `200 OK` | Event `code` | `type` | Trigger | Source | | --- | --- | --- | --- | | `stream_error` | `stream_error` | The upstream provider emitted an error chunk mid-stream. | **Provider** | -| `stream_timeout` | `timeout_error` | No chunk arrived within the idle window (2 minutes by default). | **Gateway** | +| `stream_timeout` | `timeout_error` | No chunk arrived on the gateway's internal channel within the idle window (2 minutes by default). | **Gateway** | Example terminal error event: diff --git a/docs/api-reference/overview.mdx b/docs/api-reference/overview.mdx index c855f74..3654654 100644 --- a/docs/api-reference/overview.mdx +++ b/docs/api-reference/overview.mdx @@ -1,10 +1,10 @@ --- title: API overview -description: OpenAI-compatible REST API endpoints exposed by the Ferro Labs AI Gateway — chat completions, embeddings, image generation, model listing, and streaming responses. +description: OpenAI-compatible REST API exposed by the Ferro Labs AI Gateway — chat, embeddings, images, audio, rerank, moderations, responses, and health endpoints. keywords: [AI gateway API, OpenAI compatible API, LLM REST API, chat completions API, gateway streaming API] --- -The gateway exposes an OpenAI compatible API surface so you can reuse existing clients. +The gateway exposes an OpenAI-compatible REST API surface so you can reuse existing OpenAI SDKs and tooling — point the client's `base_url` at the gateway and keep your code unchanged. ## Base URL @@ -14,13 +14,73 @@ Use the gateway host as the base URL, for example: http://localhost:8080 ``` -## Common endpoints +## Authentication -- `/v1/chat/completions` -- `/v1/models` -- `/v1/embeddings` -- `/v1/images/generations` -- `/health` -- `/metrics` +`/v1/*` requires a bearer token by default: -Other `/v1/*` endpoints (for example `/v1/audio/*`, `/v1/files`, `/v1/realtime`) are proxied to the selected provider while preserving payloads. +``` +Authorization: Bearer fgw_... +``` + +Set `ALLOW_UNAUTHENTICATED_PROXY=true` to disable this for local development — it is refused at startup under `GATEWAY_ENV=production`. See [Authentication](/guides/auth). + +## Inference endpoints + +These route through the gateway's targets, plugins, and circuit breakers like any other request: + +| Endpoint | Method | Notes | +|---|---|---| +| `/v1/chat/completions` | POST | Supports `stream: true` (SSE) | +| `/v1/completions` | POST | Legacy text completions, served as a single-message chat | +| `/v1/embeddings` | POST | | +| `/v1/images/generations` | POST | | +| `/v1/audio/speech` | POST | Text-to-speech — JSON in, binary audio out | +| `/v1/audio/transcriptions`, `/v1/audio/translations` | POST | Speech-to-text — multipart upload, 25 MiB cap | +| `/v1/rerank` | POST | Cohere-v2 contract | +| `/v1/moderations` | POST | | +| `/v1/responses` | POST | Governed and **priced** — routes by model like chat | + +`/v1/audio/*` is natively routed, not blindly proxied: it goes through targets, plugins, the circuit breaker, and request logging like the endpoints above. + +## Discovery + +| Endpoint | Method | Notes | +|---|---|---| +| `/v1/models` | GET | Union of catalog, live discovery, and operator-declared models | +| `/v1/capabilities` | GET | Per-provider OpenAI parameter support, from the capability matrix | + +## Files, batches, and responses sub-routes + +| Endpoint | Method | Notes | +|---|---|---| +| `/v1/files`, `/v1/files/*` | GET, POST, DELETE | Pass-through to the single configured `batch_target` | +| `/v1/batches`, `/v1/batches/*` | GET, POST, DELETE | Pass-through to the single configured `batch_target` | +| `/v1/responses/*` | GET, POST, DELETE | Id sub-routes (retrieve/delete/cancel/input_items), pin to `responses_target` | + +These carry no model — a batch or response id is opaque and provider-scoped — so they pin to one configured target instead of being routed. Each returns `501` when its target (`batch_target` / `responses_target`) is unset. + +## Pass-through proxy + +Any other `/v1/*` path is forwarded transparently to the provider that owns the request's model, resolved through the same routing index every native endpoint uses — an unowned model is refused `404 model_not_found` rather than forwarded. + +## Health and observability + +| Endpoint | Method | Auth | +|---|---|---| +| `/livez` | GET | None — process is alive | +| `/readyz` | GET | None — gateway can route traffic | +| `/health` | GET | None — deep diagnostic (per-provider status, circuit state) | +| `/metrics` | GET | Bearer token, `read_only` or `admin` scope | +| `/debug/vars`, `/debug/pprof/*` | GET | Bearer token, `admin` scope (`/debug/pprof/*` also requires `ENABLE_PPROF=true`) | + +:::note +`/metrics` and everything under `/debug` require a scoped bearer token — unlike `/v1/*`, an unauthenticated request always gets `401` here, even with `ALLOW_UNAUTHENTICATED_PROXY=true`. +::: + +## Related + +- [Endpoints](/api-reference/endpoints) — full request/response reference +- [Streaming](/api-reference/streaming) — SSE wire format +- [Admin API](/api-reference/admin) — key management, config history, logs +- [Errors](/api-reference/errors) — error envelope and status codes +- [Interactive API reference](/api) — Scalar-rendered OpenAPI explorer diff --git a/docs/api-reference/streaming.mdx b/docs/api-reference/streaming.mdx index e71f317..135c2d7 100644 --- a/docs/api-reference/streaming.mdx +++ b/docs/api-reference/streaming.mdx @@ -1,7 +1,7 @@ --- title: Streaming (SSE) contract -description: How the Ferro Labs AI Gateway streams chat completions over Server-Sent Events — stream mode, the [DONE] terminator, mid-stream error framing, and idle timeouts. -keywords: [AI gateway streaming, server-sent events, SSE stream, stream_error event, stream_timeout, OpenAI compatible streaming] +description: "How the Ferro Labs AI Gateway streams chat completions over SSE — chunk format, the usage frame, error events, timeouts, and MCP tool-loop streaming behavior." +keywords: [AI gateway streaming, server-sent events, SSE stream, stream_error event, stream_timeout, OpenAI compatible streaming, MCP streaming] --- The gateway streams chat completions using [Server-Sent Events (SSE)](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events). The wire format is OpenAI-compatible: each event is a `data:` line carrying one JSON chunk, and a normal stream ends with a literal `data: [DONE]` sentinel. @@ -42,14 +42,22 @@ data: {"id":"chatcmpl-abc","object":"chat.completion.chunk","created":1718600000 data: {"id":"chatcmpl-abc","object":"chat.completion.chunk","created":1718600000,"model":"gpt-4o-mini","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]} +data: {"id":"chatcmpl-abc","object":"chat.completion.chunk","created":1718600000,"model":"gpt-4o-mini","choices":[],"usage":{"prompt_tokens":12,"completion_tokens":4,"total_tokens":16}} + data: [DONE] ``` The blank line after each `data:` event is part of the SSE framing — do not strip it before your parser sees the event boundary. +## Terminal usage frame + +The gateway asks the upstream provider for a usage-carrying terminal chunk on every OpenAI-compatible stream — it sets `stream_options.include_usage: true` unless the client's own request already configured `stream_options`. That final chunk carries `choices: []` (never `null`; a usage-only frame still serializes an empty array so a client that unconditionally indexes `choices[0]` does not break) and a populated `usage` object. + +The gateway needs real token counts for metering, cost accounting, and the budget plugin regardless of what the client asked for, so it always requests the frame from the provider. What reaches the client is separate: if the original request set `stream_options.include_usage: false`, the gateway still collects the real usage internally but strips the `usage` field from the copy of the chunk it forwards, so an explicit client opt-out is honored without losing gateway-side accounting. + ## Mid-stream errors -If the upstream provider fails *after* the stream has started, the gateway cannot reuse a normal HTTP status code (headers are already sent). Instead it emits a single error event as a `data:` line and then closes the stream. No `[DONE]` sentinel follows an error event. +If the upstream provider fails *after* the stream has started, the gateway cannot reuse a normal HTTP status code (headers are already sent). Instead it emits a single error event as a `data:` line and then closes the stream. No `[DONE]` sentinel follows an error event. The failure is also recorded on the request's span, counted in Prometheus metrics, and written to the request log via the `on_error` plugin stage — a mid-stream failure is no longer a silent truncation on any of those surfaces. The error chunk has this exact JSON shape: @@ -99,15 +107,23 @@ Every SSE event is a `data:` line. Clients should classify each event as follows 1. If the payload is the literal `[DONE]`, the stream completed normally — stop reading. 2. Otherwise parse the payload as JSON, then: - A **top-level `error` object** means a failure event. Read `error.code` (`stream_error` or `stream_timeout`) and `error.message`, surface it, and stop. - - A payload with `object: "chat.completion.chunk"` (and a `choices` array) is a **normal token chunk**. Append `choices[].delta.content`. + - A payload with `object: "chat.completion.chunk"` and a non-empty `choices` array is a **normal token chunk**. Append `choices[].delta.content`. + - A payload with `object: "chat.completion.chunk"` and an **empty** `choices` array is the terminal usage frame — read `usage` and otherwise ignore it; it carries no content to append. -The presence of a top-level `error` key is the reliable discriminator: normal chunks never carry one, and error events never carry `choices`. +The presence of a top-level `error` key is the reliable discriminator: normal and usage chunks never carry one, and error events never carry `choices`. ## MCP-augmented streaming -When [MCP](https://modelcontextprotocol.io) tool servers are configured, a streaming request still accepts `stream: true`, but the gateway runs the full agentic tool-call loop to completion *before* sending anything to the client. Intermediate tool-loop turns are forced to non-streaming so each response can be inspected for `tool_calls`, so those intermediate tokens are never streamed. +Whether [MCP](https://modelcontextprotocol.io) tool servers change the streaming behavior of a request depends on whether the *request itself* carries a `tools` array: + +- **The request sends its own `tools`.** MCP does not participate at all — the gateway never advertises its MCP tools alongside a caller-supplied set, because a model could then return a mix of gateway-owned and caller-owned tool calls that neither side alone can execute. Such a request passes straight through the ordinary streaming path described above, untouched, whatever MCP servers are configured. +- **The request sends no `tools`, and the gateway has MCP tools ready to advertise.** The gateway injects its MCP tool definitions and the request is diverted: the full agentic tool-call loop (call the tool, feed the result back, re-ask the model) runs to completion **before anything is sent to the client**. Every intermediate turn is forced to non-streaming so its response can be inspected for `tool_calls`. Once the loop produces a final answer with no more pending tool calls, that answer is wrapped into a **single SSE chunk** followed by `data: [DONE]` — the client still receives `stream: true` semantics (SSE framing, one `[DONE]` terminator), but gets one content chunk rather than an incremental token stream. + +Gating is on tools *actually being advertised* (registered **and** past their MCP handshake), not merely on `mcp_servers` being configured — a server that failed to initialize does not silently collapse every tool-less stream on the gateway into a buffered response. + +**Every turn of the loop runs the request's guardrail, rate-limit, and budget plugins** — not only the turn the caller made. A tool result returned by an external MCP server is content the caller never wrote, so it is inspected exactly as the original prompt was before being sent to the model again. This means a loop can be rejected mid-way through, most commonly by the budget plugin once accumulated per-turn spend crosses its cap. -The final assistant message is wrapped into a **single SSE chunk** followed by `data: [DONE]`. In other words, with MCP active the client receives one content chunk rather than an incremental token stream. (This is the current Phase 1 behavior; true final-response token streaming is planned.) +A mid-loop rejection is **not** an SSE error event: because the loop runs entirely before any bytes reach the client, a rejection happens before the gateway ever opens the SSE response, so the caller gets an ordinary (non-streaming) JSON error — for a budget rejection, `402 insufficient_quota` — instead of a `stream_error` frame. ## Notes diff --git a/docs/benchmarks.mdx b/docs/benchmarks.mdx index 8830e21..932a2a7 100644 --- a/docs/benchmarks.mdx +++ b/docs/benchmarks.mdx @@ -1,6 +1,6 @@ --- title: Performance Benchmarks -description: "AI gateway benchmark results: sub-millisecond p99 latency overhead at 500 RPS. Go-native AI Gateway vs Python-based LLM proxy performance." +description: "AI gateway benchmark results: sub-millisecond p99 latency overhead at 500 RPS vs Python-based LLM proxies, with full open-source reproduction steps." keywords: - AI gateway benchmark - LLM proxy performance @@ -39,6 +39,23 @@ All tests run five times; tables report median of medians (p50) and median of p9 ::: +:::warning Gateway version and config not recorded + +The original benchmark report does not state which gateway version or runtime +configuration produced the numbers below, and doesn't say whether auth +(`ALLOW_UNAUTHENTICATED_PROXY`), the per-IP rate limiter (`RATE_LIMIT_RPS`, +**on by default since v1.1.18** at 20 rps / burst 40), or the request-logger +plugin were enabled during the run. All three sit on the request hot path and +add measurable overhead, and AI Gateway's current shipping defaults — auth +required on `/v1/*`, per-IP limiting on, plus the v1.4.0 unified routing +pipeline — may not match what was measured here. + +Treat these numbers as directional rather than a v1.4.x guarantee. [Reproduce +the benchmarks](#reproduce-the-benchmarks) against the version and config you +plan to run to get numbers you can cite. + +::: + --- ## Ferro Labs AI Gateway vs Python-Based Alternatives at 500 RPS @@ -80,7 +97,7 @@ Sub-millisecond p99 overhead holds through 1 000 RPS. Even at 2 000 RPS the gate - **Goroutines for concurrency** — thousands of in-flight requests multiplexed onto a small thread pool with near-zero scheduling cost. No thread-per-request overhead. - **No GIL** — every CPU core does real work in parallel. Python's Global Interpreter Lock serializes CPU-bound gateway logic (auth, routing, logging) across all requests. - **Low-GC overhead with careful allocation** — arena-style buffering and sync.Pool reuse keep heap churn minimal. GC pauses stay under 0.5 ms even at 2 000 RPS. -- **Single static binary, zero dependencies at runtime** — no interpreter, no virtualenv, no pip install at deploy time. One `COPY` in your Dockerfile. +- **Single static binary** — no interpreter, no virtualenv, no pip install at deploy time. One `COPY` in your Dockerfile. (Not zero-dependency at runtime: the gateway fetches the model catalog on startup by default — set `FERRO_MODEL_CATALOG_TIMEOUT=0` for air-gapped deploys — and optional stores like Postgres are runtime dependencies you configure, not compiled in.) ::: @@ -88,7 +105,7 @@ Sub-millisecond p99 overhead holds through 1 000 RPS. Even at 2 000 RPS the gate ## Reproduce the Benchmarks -The full benchmark suite is open source. Clone it, run it, verify every number on this page. This is a LiteLLM alternative performance comparison you can audit yourself. +The full benchmark suite is open source. Clone it, run it, verify every number on this page. This is a LiteLLM alternative performance comparison you can audit yourself — pin the gateway image to the version you care about (e.g. `ghcr.io/ferro-labs/ai-gateway:v1.4.1`) since, as noted above, these numbers have not been re-measured against the current release. ```bash # Clone the benchmark repository @@ -124,4 +141,4 @@ python3 plot.py results/ - [Why Ferro Labs AI Gateway?](/guides/why-ferro) — architecture decisions behind these numbers - [Quickstart](/getting-started/quickstart) — deploy AI Gateway in under 5 minutes -- [Routing policies](/guides/routing-policies) — configure the routing layer benchmarked above +- [Routing](/routing) — configure the routing layer benchmarked above diff --git a/docs/changelog.mdx b/docs/changelog.mdx index 4fc2115..a5d9407 100644 --- a/docs/changelog.mdx +++ b/docs/changelog.mdx @@ -1,10 +1,128 @@ --- title: Changelog -description: "Release history for Ferro Labs AI Gateway — v1.1.0 adds opt-in OpenTelemetry tracing on top of the stable v1.0.0 release with 30 providers, 8 routing strategies, MCP streaming, content-based routing, A/B testing, and budget controls." -keywords: [AI gateway changelog, LLM proxy releases, AI gateway version history, AI gateway v1.1.0, OpenTelemetry tracing, gateway updates, MCP integration release] +description: "Release history for Ferro Labs AI Gateway v0.4.5 through v1.4.1: the v1.4.0 breaking routing pipeline, embedded dashboard, and native rerank/audio endpoints." +keywords: [AI gateway changelog, LLM proxy releases, AI gateway version history, AI gateway v1.4.1, AI gateway v1.4.0 breaking changes, embedded dashboard, unified routing pipeline, MCP stdio transport, OpenTelemetry tracing, gateway updates] --- -Full release notes are also on [GitHub Releases](https://github.com/ferro-labs/ai-gateway/releases). +Full release notes are also on [GitHub Releases](https://github.com/ferro-labs/ai-gateway/releases). **v1.4.1 is the latest tag.** + +## v1.4.1 — 2026-08-07 — Dependency security patch + +A dependency-security patch for the web toolchain and embedded dashboard — no gateway code changes. Clears all 12 open Dependabot alerts (PR #392). + +### What's new in v1.4.1 + +- **react-router 8.3.0** — absorbs `react-router-dom`, whose 7.x line was flagged by scanners. The client routing API is unchanged and browser test suites pass unmodified. +- **React 19.2.8**. +- **Seven build/tooling-time packages patched** — `undici`, `ip-address`, `fast-uri`, `postcss`, `@hono/node-server`, `brace-expansion`, `hono`. None of these ship inside the embedded dashboard bundle; `npm audit` reports zero vulnerabilities. + +No behavior changes, no breaking changes. + +--- + +## v1.4.0 — 2026-08-07 — One routing pipeline, endpoint parity, and the embedded dashboard + +A breaking release (PR #365; merged 2026-08-06, tag and signed release published 2026-08-07). Chat, streaming, embeddings, and image generation had each grown their own copy of retry, circuit breaking, and error classification; this release puts all four on one routing pipeline, so a behavior is either true of every surface or of none. It also widens the native API surface and ships the operations dashboard embedded in the OSS binary — not as a standalone artifact. + +### What's new in v1.4.0 + +- **One routing pipeline** — chat, streaming, embeddings and images now share `routeTargets`; retry, circuit breaking, concurrency limits, error classification, metrics, and request logging are wired once. See [Routing](/routing). +- **Embedded dashboard** — the operations dashboard is a React SPA built into the binary and served at the gateway's own root, on the same port. One artifact, one origin, no separate container. +- **Native endpoint parity** — `POST /v1/rerank` (Cohere v2 contract), `POST /v1/moderations`, and `POST /v1/audio/{transcriptions,translations,speech}` become routed surfaces carrying the full gateway lifecycle (targets, strategy, plugins, circuit breaker, concurrency, metrics, request logging) instead of depending on the generic pass-through. +- **Files and Batches** — `/v1/files*` and `/v1/batches*` forward to one configured `batch_target` (`501` when unset). **Responses API** — `/v1/responses` is now governed and priced, with usage teed from the response body or the terminal SSE event; its stateful sub-routes pin to `responses_target`. +- **`targets[].models`** — operators can declare models a target serves that neither the catalog nor live discovery see; additive only, wildcards rejected at load. +- **Admin sessions** — `POST`/`DELETE /admin/session`, session listing, and per-session revocation. The admin key is no longer kept in the browser (24h absolute / 1h idle). +- **New admin surface** — `GET /admin/audit` (durable audit trail), `GET /admin/plugins/catalog`, `GET /admin/logs/stats` (p50/p95/p99 latency + TTFT, token split, spend per provider/model). Request-log rows gain `duration_ms`, `ttft_ms`, `cost_usd`, `api_key_id`. +- **Agentic MCP loops now bill and guard every turn** — a budget can stop an overspending loop mid-request, and a loop that fails part-way is still billed for what it spent. See [MCP](/guides/mcp). +- **More surfaces per provider** — image generation on gemini, deepinfra, and together; embeddings on azure-foundry; 17 streaming providers get a 120s response-header timeout. +- Dashboard Tracing page and a fullstack observability demo compose stack under `deploy/`; license notices ship in every release artifact; container images publish as one multi-platform manifest with an SPDX SBOM attestation. + +### Breaking changes in v1.4.0 + +- **`targets[].retry` is now honored under every routing mode**, not only `fallback` — set `attempts: 1` on a target to keep the old single-attempt behavior. This is the change most deployments will notice. +- **`targets` is an allowlist on every surface** — a provider that's registered but not listed under `targets` serves nothing; an unowned model returns `404 model_not_found`. Pool modes (fallback, loadbalance, least-latency, cost-optimized, ab-test) advance past a failed target; named modes (single, conditional, content-based) still stop and report. Every mode now skips an open circuit — all targets open returns `503`, not `404`. +- **Exhausted budget returns `402 insufficient_quota`**, not `429` — the OpenAI SDKs don't retry `402`; `429` stays reserved for rate limiting and concurrency backpressure. +- **Ollama's models variable is renamed `FERRO_OLLAMA_MODELS`** — the old `OLLAMA_MODELS` name warns this release and is removed next. +- **`ADMIN_BOOTSTRAP_KEY`** **and its companions are removed** — set `MASTER_KEY` instead; `ferrogw init` generates one. +- **`/metrics` requires the `read_only` or `admin` scope; `/debug/*` requires `admin`** — a valid bearer credential is no longer enough on its own. +- **`_BASE_URL` is the API root used verbatim on every provider** (eight providers changed meaning) — a value with a path must include the version segment. +- **Plugin API: `Context.Skip`** **is removed** — `SkipProvider` replaces it and skips only the provider call. Every remaining plugin, including `after_request`, still runs, so a cache hit can no longer disable a guardrail behind it. Pre-1.4.0 third-party plugins do not compile. +- **Container images publish as one multi-platform manifest** — the `-amd64`/`-arm64` tags are gone. +- **`/dashboard`** **, `/dashboard/*`, and `/logo.png` are removed** — the embedded SPA at the site root serves unmatched paths (no `410`). + +See [Plugins](/plugins), [Routing](/routing), and [Provider configuration](/providers/configuration) for the updated contracts. + +--- + +## v1.3.2 — 2026-07-21 — Disclosure and durability patch + +Folds in disclosure and durability fixes originally scoped as a separate patch — zero breaking changes. + +### What's new in v1.3.2 + +- **Provider error text is filtered before it leaves the process** — a credential echoed by an upstream can no longer reach clients, logs, SSE frames, or exporters. Best-effort pattern matching reduces exposure; it does not eliminate it. +- **A crashed MCP stdio subprocess is detected** and its tools withdrawn from the model (stdio only — a dead HTTP MCP server after handshake is not detected). +- **Admin config changes and their audit/history entries are applied and recorded as one serialized operation**, so a rollback can no longer target the wrong version. +- **`mcp_servers[].required`** (opt-in, default `false`) gates `/readyz` on a server's availability; the `/readyz` body gains per-server MCP state (the failure reason is deliberately omitted from the unauthenticated endpoint). +- **New MCP observability** — `gateway_mcp_server_up` gauge, `gateway_mcp_server_init_failures_total` counter, and tracing spans for MCP startup. +- MCP tools that return an error result are now recorded as errors (metrics/audit/spans), not successes. + +## v1.3.1 — 2026-07-21 — Seven post-1.3.0 fixes + +Seven defects from v1.3.0 fixed, zero breaking changes — much of it configuration that was quietly not being applied. + +### What's new in v1.3.1 + +- **Streaming now honors retry and fallback** (start-of-stream only — a begun stream is never replayed); retry backoff is unified across every surface (previously zero outside chat). +- **Embeddings and image generation now follow the configured routing strategy** — previously first-capable-provider with no fallback, retry, or timeout — and now emit metrics, cost, tracing spans, and lifecycle events. +- `/v1/completions` accepts its documented request shapes (array/token-id prompt, string `stop`). +- Circuit breaker made panic-safe — a panicking provider call no longer disables its target until restart; provider lookup order is now stable. +- `stream_options.include_usage` is honored; API-key updates no longer apply changes they rejected. +- **Model catalog is now actually downloaded** — fetch budget raised from 1s to 10s — plus `FERRO_MODEL_CATALOG_TIMEOUT` (`0` skips the fetch for air-gapped deployments). + +No breaking API changes, but two operationally visible effects: which provider serves embeddings/images can shift under non-single strategies (worth a cost/latency/data-residency review), and reported costs move to pricing from the live model catalog rather than the embedded snapshot. + +## v1.3.0 — 2026-07-20 — MCP stdio transport + +Adds a second MCP transport and closes an environment-leak risk in the first one. + +### What's new in v1.3.0 + +- **MCP stdio transport** — `mcp_servers` entries may set `command` (+ `args`) instead of `url`, running any npx/uvx/binary MCP server as a gateway-launched subprocess. Contributed by @gr3enarr0w (PR #121). See [MCP](/guides/mcp). +- **Subprocess environment isolation** — an MCP subprocess inherits no gateway environment; only `PATH`/`HOME`/`LANG`/`TMPDIR` plus that server's own `env` block reach it. +- `${VAR}` references resolve in a stdio server's `env` at client construction; `GET /admin/config` redacts `env` alongside `headers`. +- stderr is drained into the gateway log at debug level, so a startup failure is readable instead of an opaque timeout. +- Fixed: a misconfigured MCP server no longer disables streaming gateway-wide — activation now keys off discovered tools, not registered servers. +- Fixed: caller-supplied tool calls are no longer intercepted — client-side OpenAI function calling works with MCP enabled. +- Fixed: tool calls are capped per turn, npx grandchild processes are reaped via process groups, and heavy stderr writers no longer deadlock. + +No breaking changes are formally declared, but one behavior changed: MCP tools are advertised only when the request carries no tools of its own — a request sending its own `tools` array passes through untouched, unaffected by MCP entirely. + +--- + +## v1.2.0 — 2026-07-14 — Provider capability matrix and plugin failure policy + +Formalizes provider capability declarations and fixes a dangerous plugin-error contract: a broken plugin no longer masquerades as a clean deny. + +### What's new in v1.2.0 + +- **`GET /v1/capabilities` + provider capability matrix** — a declarative source of which OpenAI chat parameters each provider forwards, translates, or cannot express, enforced rather than merely advertised. +- **`compatibility.on_unsupported_param`**: `warn` (default) | `drop` | `reject` — unsupported parameters are no longer silently discarded. +- **`request_timeout`** bounds a non-streaming request end-to-end (plugins + provider call + every retry/fallback); streaming is exempt except on the MCP agentic path. +- **`targets[].concurrency`** — per-target in-flight limits with a bounded queue; saturation sheds `429 provider_saturated`. +- **`/livez` and `/readyz`** split from `/health` for orchestrator gating. +- **Cross-provider conformance suite** (`test/conformance/`) — native-payload fixtures, no network; new providers must add a fixture or a declared exemption. +- A hung provider now trips its circuit breaker (the gateway attributes the deadline to the provider). + +### Breaking changes in v1.2.0 + +- **Plugin authors: deny by verdict, not error** — a returned error now means the plugin broke and yields `500 plugin_error`; `Context.Reject` is the only way to deny a request. A down rate-limit backend no longer returns `429` and triggers SDK retry storms. +- `Context.Reject` is now honored for every plugin type (previously silently discarded for logging/metrics/transform plugins). +- Transform plugins fail closed on error (previously fail-open). +- **Retries limited to the default retryable set** (408/429/5xx) with exponential backoff and full jitter, honoring `Retry-After` capped at 30s, stopping at the deadline — `400`/`401` are no longer retried. +- **`${VAR}` substitution**: only the braced form is a reference; a bare `$` is literal data (`$100`, `pa$$w0rd` survive); an undefined variable is now a startup error; resolution moved from config load to component construction so secrets never enter config history/rollback. + +--- ## v1.1.0 — 2026-05-24 — OpenTelemetry tracing diff --git a/docs/enterprise.mdx b/docs/enterprise.mdx index 5b16ef7..6f08561 100644 --- a/docs/enterprise.mdx +++ b/docs/enterprise.mdx @@ -1,6 +1,6 @@ --- title: Enterprise -description: "Ferro Labs Managed enterprise features for the Ferro Labs AI Gateway — SSO/SAML, RBAC, audit logs, multi-region deployment, compliance packs, and dedicated support for production teams." +description: "Ferro Labs Managed enterprise features for the AI Gateway: SSO/SAML, RBAC, SIEM audit export, multi-region deployment, compliance packs, dedicated support." keywords: [enterprise AI gateway, enterprise LLM proxy, enterprise AI infrastructure, SSO AI gateway, on-premise LLM, dedicated AI support] sidebar_position: 99 --- @@ -161,14 +161,16 @@ tuning, load testing, and runbook creation by a Ferro Labs solutions engineer. | Community support | ✅ | ✅ | | SSO / SAML 2.0 | — | ✅ | | Advanced RBAC | — | ✅ | -| Audit logs + SIEM export | — | ✅ | +| Audit logs (durable trail, dashboard Audit page) | ✅ | ✅ | +| SIEM export (Splunk, Datadog, OpenSearch) | — | ✅ | | SLA with dedicated support | — | ✅ | | Multi-region & HA deployment | — | ✅ | | Custom plugins | — | ✅ | | Compliance packs (SOC2, GDPR …) | — | ✅ | | Kubernetes Operator | — | ✅ | | On-premises / air-gapped | — | ✅ | -| Advanced analytics + cost attribution | — | ✅ | +| Analytics (traffic, latency/TTFT, cost by provider/model/key) | ✅ | ✅ | +| Long-retention analytics (ClickHouse-backed) | — | ✅ | | White-labeling | — | ✅ | | Dedicated onboarding | — | ✅ | diff --git a/docs/faq/index.mdx b/docs/faq/index.mdx index db7606c..e45835f 100644 --- a/docs/faq/index.mdx +++ b/docs/faq/index.mdx @@ -1,12 +1,46 @@ --- title: FAQ -description: Frequently asked questions about the Ferro Labs AI Gateway — setup, provider compatibility, streaming support, Docker deployment, routing strategies, and plugin configuration. +description: Answers on gateway auth, provider setup, routing, plugins, MCP, and budgets for the Ferro Labs AI Gateway — an open-source OpenAI-compatible LLM proxy. keywords: [AI gateway FAQ, LLM proxy questions, OpenAI compatible FAQ, AI gateway help, gateway troubleshooting] hide_table_of_contents: true --- import styles from './faq.module.css'; + +
# Frequently Asked Questions @@ -19,14 +53,23 @@ import styles from './faq.module.css'; Do I need to change my SDK or application code?
-No. The gateway uses the OpenAI wire format. Change only `base_url` in your client: +No. The gateway uses the OpenAI wire format. Change `base_url` and pass a bearer token as `api_key`: ```python from openai import OpenAI -client = OpenAI(base_url="http://localhost:8080/v1", api_key="any") +client = OpenAI(base_url="http://localhost:8080/v1", api_key="fgw_...") ``` -The `api_key` field is required by the OpenAI SDK but the gateway ignores it — provider credentials are set via environment variables on the server. +Unlike many proxies, the gateway does **not** ignore `api_key` — every `/v1/*` route requires a bearer token by default (`MASTER_KEY` or an issued `fgw_...` key). A placeholder value like `api_key="any"` gets a `401`. Auth can be disabled for local development only with `ALLOW_UNAUTHENTICATED_PROXY=true`; it is refused when `GATEWAY_ENV=production`. See [Authentication](/guides/auth). + +
+ + +
+Is there a web UI? +
+ +Yes. A dashboard is embedded directly in the gateway binary and served at the root path (`/`) — there is no separate container, image, or `GATEWAY_BASE_URL` to configure. It covers API key management, request logs, config history, provider health, analytics, and an in-browser Playground that authenticates with the same session token the admin API uses. See the [Dashboard guide](/guides/dashboard).
@@ -35,7 +78,7 @@ The `api_key` field is required by the OpenAI SDK but the gateway ignores it — Which model names should I use?
-Use the model IDs native to each provider — `claude-3-5-sonnet-20241022` for Anthropic, `gemini-1.5-pro` for Gemini, `gpt-4o` for OpenAI, etc. You can also define [model aliases](/getting-started/configuration#model-aliases) like `fast` or `smart` in your config to decouple your application from specific model names. +Use the model IDs native to each provider — `claude-3-5-sonnet-20241022` for Anthropic, `gemini-1.5-pro` for Gemini, `gpt-4o` for OpenAI, and so on. You can also define [model aliases](/getting-started/configuration#model-aliases) to decouple your application from a specific provider's model names, or use `targets[].models` to [declare a model](/getting-started/configuration#declared-models) the gateway can't otherwise discover.
@@ -54,6 +97,8 @@ docker run -p 8080:8080 -e OPENAI_API_KEY=sk-... ghcr.io/ferro-labs/ai-gateway:l git clone https://github.com/ferro-labs/ai-gateway && cd ai-gateway && make run ``` +The image `ghcr.io/ferro-labs/ai-gateway` is a single multi-platform manifest — there are no separate `-amd64`/`-arm64` tags to choose between. +
@@ -61,7 +106,7 @@ git clone https://github.com/ferro-labs/ai-gateway && cd ai-gateway && make run Does it support streaming?
-Yes. Set `stream: true` in your request body. The gateway streams the response from the provider to your client using Server-Sent Events (SSE), identical to the OpenAI streaming format. This works across all supported providers. +Yes. Set `stream: true` in your request body. The gateway streams the response using Server-Sent Events, identical to the OpenAI streaming format, across chat completions and every provider that supports it.
@@ -70,7 +115,7 @@ Yes. Set `stream: true` in your request body. The gateway streams the response f What Go version is required?
-Go 1.21 or later. The gateway binary is statically compiled and has no runtime dependencies. Pre-built binaries and a Docker image are available on the [GitHub releases page](https://github.com/ferro-labs/ai-gateway/releases). +Go 1.25 or later to build from source. The gateway binary is a single static binary; pre-built binaries and a multi-platform Docker image (`ghcr.io/ferro-labs/ai-gateway`) are available on the [GitHub releases page](https://github.com/ferro-labs/ai-gateway/releases).
@@ -85,14 +130,14 @@ Go 1.21 or later. The gateway binary is statically compiled and has no runtime d How do I enable a provider?
-Set the provider's environment variable before starting the gateway. For example: +Set the provider's required environment variable(s) before starting the gateway — a provider is auto-registered only when its credential env var is present: ```bash export ANTHROPIC_API_KEY=sk-ant-... export GROQ_API_KEY=gsk_... ``` -No config file changes are needed to enable providers — setting the environment variable is sufficient. See [Provider configuration](/guides/providers-config) for all variables. +Registering a provider isn't the whole story: it also needs a `targets[].virtual_key` entry naming it in your config, or requests to it are never routed. See [Provider configuration](/providers/configuration).
@@ -101,7 +146,7 @@ No config file changes are needed to enable providers — setting the environmen Can I use multiple providers at the same time?
-Yes. Set credentials for as many providers as you want. The gateway discovers enabled providers at startup based on which environment variables are set. Use [routing strategies](/guides/routing-policies) to control which provider gets each request — combine multiple providers in a single route for fallback, load balancing, or cost optimization. +Yes. Set credentials for as many providers as you want, then list each as a `targets[]` entry with a routing `strategy`. `targets` is an allowlist on every routed surface — a registered provider that isn't listed in `targets[]` returns `404 model_not_found` for its models. See [Routing](/routing) for fallback, load-balance, and cost-optimized strategies.
@@ -110,7 +155,9 @@ Yes. Set credentials for as many providers as you want. The gateway discovers en Can I use Ollama or other self-hosted models?
-Yes. Set `OLLAMA_HOST=http://localhost:11434` and `OLLAMA_MODELS=llama3.2,mistral`. Ollama requires no API key. The gateway supports any provider that exposes an OpenAI-compatible HTTP API via the `openai-compatible` provider type — set `CUSTOM_PROVIDER_BASE_URL` and optionally `CUSTOM_PROVIDER_API_KEY`. +Yes. Set `OLLAMA_HOST=http://localhost:11434` (Ollama's server root — no API key needed) and, optionally, `FERRO_OLLAMA_MODELS=llama3.2,mistral` to narrow which models `/v1/models` advertises. The older `OLLAMA_MODELS` variable is deprecated and scheduled for removal — it's Ollama's own models-*directory* variable, and the gateway drops a path-shaped value with a warning rather than misreading it as a model list. + +For any other OpenAI-compatible server, point the matching provider's `_BASE_URL` override at it (write the API root verbatim, version segment included).
@@ -119,15 +166,15 @@ Yes. Set `OLLAMA_HOST=http://localhost:11434` and `OLLAMA_MODELS=llama3.2,mistra Can I force a specific provider for one request?
-Yes. Add the `X-Provider` header to your request: +Only on the pass-through proxy. `X-Provider` resolves which provider a `/v1/*` request that the gateway doesn't route natively (`/v1/files`, `/v1/batches`, and similar) gets forwarded to: ```bash -curl http://localhost:8080/v1/chat/completions \ - -H "X-Provider: anthropic" \ - -d '{"model": "claude-3-5-sonnet-20241022", "messages": [...]}' +curl http://localhost:8080/v1/files \ + -H "Authorization: Bearer $MASTER_KEY" \ + -H "X-Provider: openai" ``` -This overrides the routing strategy for that single request. +On routed surfaces — chat completions, streaming, embeddings, images — `X-Provider` has no effect. Those always resolve through your configured `strategy` and `targets` allowlist; a provider not in `targets[]` is refused regardless of any header.
@@ -142,7 +189,7 @@ This overrides the routing strategy for that single request. What happens if a provider is down?
-With the `fallback` strategy, the gateway automatically retries the next configured target with exponential backoff. With `single`, the error is returned to the client immediately. Circuit breakers automatically exclude failing providers across all strategies once the error threshold is exceeded — they self-recover after the configured timeout. +Depends on the strategy. Pool modes — `fallback`, `loadbalance`, `least-latency`, `cost-optimized`, `ab-test` — advance to the next target in the pool on a failure. Named modes — `single`, `conditional`, `content-based` — report the failure back to the caller instead, since something specifically chose that target. Every mode skips a target whose circuit breaker is open; if every target for a model is open, the request gets `503`. `targets[].retry` is honored under every mode, so set `attempts: 1` to keep single-attempt behavior even in a pool.
@@ -151,16 +198,16 @@ With the `fallback` strategy, the gateway automatically retries the next configu How does cost-optimized routing work?
-The gateway ships with a catalog of 2,500+ models with input/output cost data per million tokens. For each request, it estimates the cost for each configured target based on the model name and routes to the cheapest compatible provider. No external API calls are made for cost data — it ships embedded in the binary. +The gateway ships with an embedded catalog of 2,500+ models with per-million-token input/output pricing. For each request under `mode: cost-optimized`, it estimates cost per configured target and routes to the cheapest one that's still routable. `unpriced_strategy` (`fallback` | `skip` | `allow`) controls what happens when a target's model has no catalog price.
-What is the difference between `least-latency` and `fallback`? +What's the difference between `least-latency` and `fallback`?
-`fallback` is **reactive** — it switches providers only after a failure or timeout. `least-latency` is **proactive** — it continuously measures P50 latency from successful requests and always prefers the fastest available provider, even when all providers are healthy. Use `least-latency` when minimizing response time is more important than cost. +`fallback` is **reactive** — it only moves to the next target after a failure. `least-latency` is **proactive** — it continuously tracks P50 latency from successful requests and prefers the fastest available target, even when every target is healthy.
@@ -169,7 +216,7 @@ The gateway ships with a catalog of 2,500+ models with input/output cost data pe Can I route different request types to different providers?
-Yes. The `conditional` strategy allows routing based on request attributes — model name, request headers, or custom metadata. For example, you can route requests with `X-Tier: premium` to GPT-4o and all others to a cheaper model. See [Routing policies](/guides/routing-policies) for full examples. +Yes, with the `conditional` strategy — but only on a closed set of keys, validated at config load: `model` (exact match) or `model_prefix` (prefix match). Arbitrary request headers or custom metadata aren't supported condition keys; a `conditions[]` entry naming anything else fails `ferrogw validate` before the gateway starts. For matching on prompt content instead of the model name, use `content-based`. See [Conditional routing](/routing/conditional).
@@ -181,19 +228,19 @@ Yes. The `conditional` strategy allows routing based on request attributes — m ## Plugins & Safety
-Do plugins affect latency? +What happens when I run out of budget?
-Guardrail plugins (`before_request`) add minimal overhead — typically under 1ms for pattern-matching plugins (`word-filter`, `regex-guard`). `pii-redact` and `prompt-shield` may add 1–5ms depending on content size. `response-cache` can dramatically reduce latency on cache hits by returning responses without hitting any provider. +The `budget` plugin returns `402 Payment Required` with error type `insufficient_quota` once committed spend reaches the configured limit — not `429`. It's a read-only soft-cap check (no reservation), evaluated at `before_request`, and must also be listed at `after_request` with byte-identical config so it can record spend. See [Budget plugin](/plugins/budget).
-Can I write custom plugins? +Do plugins affect latency?
-Yes. Implement the `plugin.Plugin` interface in Go and register it with the plugin manager. The interface requires `Name()`, `Type()`, `Init()`, and `Execute()` methods. See the [examples directory](https://github.com/ferro-labs/ai-gateway/tree/main/examples/custom-plugin) for a working example. +Guardrail plugins like `word-filter` and `max-token` add minimal overhead — pattern matching, not a network call. `response-cache` can dramatically *reduce* latency on cache hits by returning a response without calling any provider.
@@ -202,7 +249,16 @@ Yes. Implement the `plugin.Plugin` interface in Go and register it with the plug Will plugins block my requests in production?
-Yes, if a guardrail with `action: block` is triggered, the request is rejected immediately with a `400` or `403` response. You can also configure `action: warn` to log without blocking. Test your guardrail configuration in a staging environment and review logs at `/admin/logs` before enabling in production. +Yes, when a plugin issues a `Reject` verdict — the request gets a `4xx`, `429`, or `402` response depending on the plugin. Separately, a plugin *error* (the plugin itself breaking) fails **closed** with a `500` for guardrail, auth, rate-limit, and transform plugins, but fails **open** for logging and metrics plugins, so a broken logger doesn't take down traffic. Review request logs at `/admin/logs` before enabling a new guardrail in production. + +
+ + +
+Can I write custom plugins? +
+ +Yes. Implement the `plugin.Plugin` interface in Go and register a factory with `plugin.RegisterFactory` in an `init()` function, then add a blank import in `cmd/ferrogw/main.go`. Plugins are configured globally under top-level `plugins:` — there's no per-route plugin field. See [Plugins overview](/plugins).
@@ -217,7 +273,7 @@ Yes, if a guardrail with `action: block` is triggered, the request is rejected i What is MCP?
-Model Context Protocol (MCP) is an open standard for connecting AI models to external tools and data sources. The gateway implements MCP as a client — it connects to your MCP tool servers, injects available tools into chat completion requests, and runs the agentic tool-calling loop automatically. See the [MCP guide](/guides/mcp). +Model Context Protocol is an open standard for connecting AI models to external tools and data sources. The gateway implements MCP as a client: when `mcp_servers[]` entries are configured, it injects their tools into chat completion requests and runs the agentic tool-calling loop internally. See the [MCP guide](/guides/mcp).
@@ -226,16 +282,28 @@ Model Context Protocol (MCP) is an open standard for connecting AI models to ext Do my clients need to implement the tool loop?
-No. The gateway handles the full agentic loop internally. Your client sends a standard chat completion request and receives a final text response. All intermediate tool calls happen transparently inside the gateway. This means any OpenAI-compatible client automatically gains tool-use capability without code changes. +No. Your client sends a normal `POST /v1/chat/completions` and gets back a final text response; every intermediate tool call happens inside the gateway, bounded by `max_call_depth` (default 5) and capped at 64 tool calls per turn.
-Which MCP server implementations are supported? +Does MCP only work over HTTP?
-The gateway supports MCP servers using the 2025-11-25 Streamable HTTP transport. Popular compatible servers include the [MCP filesystem server](https://github.com/modelcontextprotocol/servers/tree/main/src/filesystem), [PostgreSQL server](https://github.com/modelcontextprotocol/servers/tree/main/src/postgres), and [Fetch server](https://github.com/modelcontextprotocol/servers/tree/main/src/fetch). See [MCP integration](/guides/mcp) for full setup instructions. +No — each `mcp_servers[]` entry sets exactly one of `url` (Streamable HTTP, the 2025-11-25 transport revision) or `command` (+`args`, a stdio subprocess). A stdio server is launched at gateway startup and kept for the gateway's lifetime. It does **not** inherit the gateway's environment — only `PATH`/`HOME`/`LANG`/`TMPDIR` plus whatever you list explicitly in its `env:` block, so credentials like `OPENAI_API_KEY` or `MASTER_KEY` never reach it implicitly: + +```yaml +mcp_servers: + - name: filesystem + command: npx + args: ["-y", "@modelcontextprotocol/server-filesystem", "/data"] + env: + SOME_TOKEN: ${SOME_TOKEN} + required: false +``` + +Setting `required: true` gates `GET /readyz` on that server's initialize handshake — a required server that's down takes the whole instance out of rotation. Guardrail and budget plugins still run on every turn of the agentic loop, not just the initial request.
@@ -250,7 +318,16 @@ The gateway supports MCP servers using the 2025-11-25 Streamable HTTP transport. How do I know which providers are active?
-Call `GET /health` for per-provider health status, latency, and model counts. Call `GET /v1/models` for all available models grouped by provider. Both endpoints are unauthenticated and suitable for readiness probes. +`GET /health` reports per-provider status, model counts, and circuit-breaker state and is unauthenticated. `GET /v1/models` lists available models, but — like every other `/v1/*` route — it requires a bearer token unless `ALLOW_UNAUTHENTICATED_PROXY=true` is set. + +
+ + +
+What's the difference between `/health`, `/livez`, and `/readyz`? +
+ +All three are unauthenticated. `/livez` just confirms the process is up. `/readyz` answers whether the gateway can serve traffic: `200 ready` when at least one configured target is routable, `503 not_ready` (reason `no routable targets`) when none are — this is the one to point a load balancer or Kubernetes readiness probe at. It also folds in any MCP server marked `required: true`. `/health` is the deep diagnostic: per-provider status, model counts, and circuit state.
@@ -259,7 +336,7 @@ Call `GET /health` for per-provider health status, latency, and model counts. Ca Is there an admin API?
-Yes. The gateway exposes an admin API at `/admin/*` for managing API keys, querying request logs, viewing provider status, and hot-reloading config. Protect it with `ADMIN_API_KEY`. See the [Admin auth guide](/guides/admin-auth) and [interactive API reference](/api). +Yes, under `/admin/*` — API key management, request logs, audit trail, config history and rollback, plugin catalog. Bootstrap it with `MASTER_KEY` (generated via `ferrogw init`), then issue per-operator `fgw_`-prefixed keys via `POST /admin/keys`. Omitting `scopes` on creation defaults to `read_only`. See [Authentication](/guides/auth) and the [interactive API reference](/api).
@@ -268,7 +345,7 @@ Yes. The gateway exposes an admin API at `/admin/*` for managing API keys, query Can I use PostgreSQL instead of SQLite?
-Yes. For request logs, set `backend: postgres` and `dsn: postgres://...` in the `request-logger` plugin config. For the admin API key store, set `STORE_BACKEND=postgres` and `STORE_DSN=postgres://...`. See [Server settings](/operations/server-settings) for all environment variables. +Yes, per store. Set `REQUEST_LOG_STORE_BACKEND=postgres` and `REQUEST_LOG_STORE_DSN=postgres://...` for request logs. For the admin API key/session/audit store, set `API_KEY_STORE_BACKEND=postgres` and `API_KEY_STORE_DSN=postgres://...`. Config history follows the same pattern with `CONFIG_STORE_BACKEND`/`CONFIG_STORE_DSN`. All three default to in-memory, which doesn't survive a restart. See [Server settings](/operations/server-settings).
@@ -277,7 +354,7 @@ Yes. For request logs, set `backend: postgres` and `dsn: postgres://...` in the Is it production-ready?
-Yes. The gateway is used in production with circuit breakers, retries with exponential backoff, Prometheus monitoring, structured request logging, and an admin API. See [Monitoring](/operations/monitoring) for recommended Prometheus alert rules and a Grafana dashboard layout. +Yes. Auth is on by default, per-IP rate limiting is on by default (20 rps / burst 40), and the gateway ships circuit breakers, retries with exponential backoff, Prometheus metrics, structured request logging, and an audit trail. Setting `GATEWAY_ENV=production` adds startup checks that refuse to boot with `ALLOW_UNAUTHENTICATED_PROXY=true` or a wildcard `CORS_ORIGINS`, and warns on an in-memory key store. See [Monitoring](/operations/monitoring).
diff --git a/docs/ferrocloud/overview.mdx b/docs/ferrocloud/overview.mdx index 73ffe64..8b3c0ac 100644 --- a/docs/ferrocloud/overview.mdx +++ b/docs/ferrocloud/overview.mdx @@ -1,12 +1,12 @@ --- title: "Ferro Labs Managed — Managed AI Gateway" -description: "Ferro Labs Managed is the managed, multi-tenant version of Ferro Labs AI Gateway. Isolated gateway instances, dashboard, billing, semantic caching, SSO, and enterprise security plugins." +description: "Ferro Labs Managed is the hosted, multi-tenant AI Gateway: isolated tenants, semantic caching, SSO/SAML, and enterprise security plugins on the OSS engine." keywords: [Ferro Labs Managed, managed AI gateway, multi-tenant LLM proxy, hosted AI gateway, enterprise AI gateway SaaS] --- # Ferro Labs Managed — Managed AI Gateway -Ferro Labs Managed is the managed, multi-tenant, hosted version of the Ferro Labs AI Gateway. Built on the same open-source engine that powers the self-hosted gateway, Ferro Labs Managed adds isolated tenant instances, a full-featured dashboard, usage-based billing, semantic caching, SSO/SAML, and enterprise security plugins — all operated by Ferro Labs so you never touch infrastructure. +Ferro Labs Managed is the hosted, multi-tenant version of the Ferro Labs AI Gateway. Built on the same open-source engine that powers the self-hosted gateway — including its embedded dashboard, per-request cost tracking, and budget enforcement — Ferro Labs Managed adds isolated tenant instances, semantic caching, SSO/SAML, cross-tenant durable billing, and enterprise security plugins, all operated by Ferro Labs so you never touch infrastructure. ## Architecture @@ -33,10 +33,14 @@ Ferro Labs Managed pricing, plans, and feature tiers will be announced when the ## How Ferro Labs Managed Differs from Self-Hosting -1. **Zero ops** — No servers to provision, no containers to update, no TLS certificates to rotate. Ferro Labs handles uptime, scaling, and upgrades. -2. **Durable billing** — Built-in usage metering, spend tracking, and budget enforcement per team or project. No need to wire up your own billing pipeline. -3. **Enterprise plugins** — Five additional guardrail plugins (`pii-redact`, `secret-scan`, `prompt-shield`, `schema-guard`, and `regex-guard`) are available exclusively on Ferro Labs Managed Pro and Enterprise plans. -4. **Team management** — Invite teammates, assign roles, and scope API keys to projects — all from the dashboard. +The self-hosted OSS gateway already ships an embedded dashboard, persists per-request cost (`cost_usd`) with spend attribution by key/provider/model, and enforces budgets with a `402 insufficient_quota` response when a limit is hit. Ferro Labs Managed builds on top of that engine rather than duplicating it — the differences are about running it hosted, at multi-tenant scale, with capabilities that don't make sense for a single self-hosted instance: + +1. **Multi-tenancy and isolation** — Each tenant gets its own isolated gateway instance and credential boundary, rather than the single shared instance a self-hosted deployment runs. +2. **Zero ops** — No servers to provision, no containers to update, no TLS certificates to rotate. Ferro Labs handles uptime, scaling, and upgrades. +3. **Semantic caching** — Similarity-based cache hits across paraphrased prompts, beyond the OSS response-cache plugin's exact-match semantics. See [Semantic cache](/ferrocloud/semantic-cache). +4. **SSO/SAML and enterprise security plugins** — SAML 2.0/OIDC sign-in, directory sync, and five additional guardrail plugins (`pii-redact`, `secret-scan`, `prompt-shield`, `schema-guard`, and `regex-guard`) available exclusively on Ferro Labs Managed Pro and Enterprise plans. +5. **Cross-tenant durable billing** — Consolidated usage metering and invoicing across teams, projects, and tenants, on top of the per-request cost tracking the OSS gateway already records locally. +6. **Team management** — Invite teammates, assign roles, and scope API keys to projects — all from the dashboard. ## Join the Waitlist diff --git a/docs/ferrocloud/semantic-cache.mdx b/docs/ferrocloud/semantic-cache.mdx index 62f5cc8..d2e9501 100644 --- a/docs/ferrocloud/semantic-cache.mdx +++ b/docs/ferrocloud/semantic-cache.mdx @@ -55,4 +55,4 @@ Semantic caching is available on the **Pro** plan and above. It is not included - [Ferro Labs Managed overview](/ferrocloud/overview) - [OSS vs Ferro Labs Managed](/guides/oss-vs-ferrocloud) -- [Plugins](/guides/plugins) +- [Plugins](/plugins) diff --git a/docs/frameworks/dspy.mdx b/docs/frameworks/dspy.mdx index 0e3f99a..03a9228 100644 --- a/docs/frameworks/dspy.mdx +++ b/docs/frameworks/dspy.mdx @@ -39,7 +39,7 @@ print(predict(question="What is a gateway?").answer) ## Optimizers across providers -The interesting story for DSPy + Ferro: compile a program against one provider, then re-compile against another by changing only the model name. Pair with the gateway's [budget plugin](/guides/plugins) to cap optimizer cost while exploring. +The interesting story for DSPy + Ferro: compile a program against one provider, then re-compile against another by changing only the model name. Pair with the gateway's [budget plugin](/plugins) to cap optimizer cost while exploring. ## Verify @@ -57,4 +57,4 @@ curl http://localhost:8080/v1/chat/completions \ ## See also - [LangChain (Python)](/frameworks/langchain-python) -- [Use cases](/guides/routing-policies) — multi-provider patterns +- [Use cases](/routing) — multi-provider patterns diff --git a/docs/frameworks/index.mdx b/docs/frameworks/index.mdx index 920e802..972b680 100644 --- a/docs/frameworks/index.mdx +++ b/docs/frameworks/index.mdx @@ -1,6 +1,6 @@ --- title: Frameworks -description: Use Ferro Labs AI Gateway with the LLM frameworks you already know — LangChain, LangGraph, LangSmith, LlamaIndex, CrewAI, Vercel AI SDK, Mastra, DSPy, and more. Drop-in compatibility, per-provider routing, and one trace_id everywhere. +description: "Use Ferro Labs AI Gateway with LangChain, LangGraph, LlamaIndex, CrewAI, Vercel AI SDK, Mastra, DSPy, and more — drop-in compatibility with unified routing." keywords: [LangChain AI gateway, LangGraph proxy, LangSmith open-source alternative, LlamaIndex gateway, CrewAI multi-provider, Vercel AI SDK gateway, AI gateway framework integration, LiteLLM alternative] sidebar_position: 1 --- diff --git a/docs/frameworks/instructor.mdx b/docs/frameworks/instructor.mdx index c125f15..10e2b45 100644 --- a/docs/frameworks/instructor.mdx +++ b/docs/frameworks/instructor.mdx @@ -1,6 +1,6 @@ --- title: Instructor -description: Use Instructor for typed Pydantic outputs with Ferro Labs AI Gateway — patch any OpenAI client pointed at the gateway and get validated structured responses across 30+ providers. +description: "Use Instructor for typed Pydantic outputs with Ferro Labs AI Gateway — patch any OpenAI client at the gateway and get validated structured responses." keywords: [Instructor gateway, Instructor proxy, Instructor multi-provider, Pydantic structured output, LLM JSON schema gateway] sidebar_position: 14 --- diff --git a/docs/frameworks/langchain-js.mdx b/docs/frameworks/langchain-js.mdx index dc0f05f..2b45d35 100644 --- a/docs/frameworks/langchain-js.mdx +++ b/docs/frameworks/langchain-js.mdx @@ -47,7 +47,7 @@ curl -i http://localhost:8080/v1/chat/completions \ `@ferro-labs-ai/sdk/langchain` will ship a `FerroChatModel` mirroring the Python adapter — typed Ferro extras (`routeTag`, `templateId`, `templateVariables`), `traceId` on `response_metadata`, native LangGraph.js compatibility. The npm scope `@ferro-labs-ai` is already reserved. -Track progress: [Ferro AI Gateway roadmap](https://github.com/ferro-labs/ai-gateway/blob/main/ROADMAP.md). +Track progress: [Ferro Labs AI Gateway roadmap](https://github.com/ferro-labs/ai-gateway/blob/main/ROADMAP.md). ## Runnable example diff --git a/docs/frameworks/langchain-python.mdx b/docs/frameworks/langchain-python.mdx index 48efefb..00c1fe9 100644 --- a/docs/frameworks/langchain-python.mdx +++ b/docs/frameworks/langchain-python.mdx @@ -1,6 +1,6 @@ --- title: LangChain (Python) -description: Use Ferro Labs AI Gateway from LangChain Python with the first-party langchain-ferrolabsai adapter — chat, embeddings, tool calling, and LangGraph all routed across 30+ providers through one endpoint. +description: "Use LangChain Python with Ferro Labs AI Gateway via the langchain-ferrolabsai adapter — chat, embeddings, tool calling, and LangGraph through one endpoint." keywords: [LangChain AI gateway, LangChain proxy, LangChain multi-provider, langchain-ferrolabsai, LangChain Anthropic, LangChain Gemini, FerroChatModel, LangChain LLM router] sidebar_position: 2 --- diff --git a/docs/frameworks/langgraph.mdx b/docs/frameworks/langgraph.mdx index ab9a417..ffc0865 100644 --- a/docs/frameworks/langgraph.mdx +++ b/docs/frameworks/langgraph.mdx @@ -1,6 +1,6 @@ --- title: LangGraph -description: Build LangGraph agents that route each node to a different best-in-class LLM provider through one Ferro Labs AI Gateway endpoint — planner on GPT-4o, coder on Claude, summarizer on Gemini, no rewrites. +description: "Build LangGraph agents that route each node to a different provider through one Ferro Labs AI Gateway endpoint — planner, coder, and summarizer on different LLMs." keywords: [LangGraph multi-provider, LangGraph proxy, LangGraph router, LangGraph Anthropic, LangGraph Gemini, LangGraph multi-LLM agent, AI gateway agent, FerroChatModel LangGraph] sidebar_position: 3 --- @@ -116,5 +116,5 @@ Most LangGraph examples either pick one provider and stay there (limiting agent - [LangChain (Python)](/frameworks/langchain-python) — full `FerroChatModel` / `FerroEmbeddings` / `FerroLLM` reference - [LangSmith](/frameworks/langsmith) — turn the per-node `trace_id`s into LangSmith runs -- [Routing policies](/guides/routing-policies) — what the gateway does behind the scenes -- [Use cases](/guides/routing-policies) — more multi-provider patterns +- [Routing policies](/routing) — what the gateway does behind the scenes +- [Use cases](/routing) — more multi-provider patterns diff --git a/docs/frameworks/langsmith.mdx b/docs/frameworks/langsmith.mdx index bfd63f2..b2c5f9d 100644 --- a/docs/frameworks/langsmith.mdx +++ b/docs/frameworks/langsmith.mdx @@ -1,6 +1,6 @@ --- title: LangSmith -description: Bridge Ferro Labs AI Gateway to LangSmith for unified LLM observability across all 30+ providers — turn one trace_id into one LangSmith run, without per-provider SDK wiring. +description: "Bridge Ferro Labs AI Gateway to LangSmith for unified LLM observability — one trace_id becomes one LangSmith run, with no per-provider SDK wiring." keywords: [LangSmith open-source alternative, LangSmith AI gateway, LangSmith multi-provider tracing, LangSmith Anthropic, LangSmith Gemini, LangSmith bridge, AI gateway observability, ferro labs langsmith] sidebar_position: 4 --- diff --git a/docs/frameworks/llamaindex.mdx b/docs/frameworks/llamaindex.mdx index 1194902..1227d6f 100644 --- a/docs/frameworks/llamaindex.mdx +++ b/docs/frameworks/llamaindex.mdx @@ -52,7 +52,7 @@ curl http://localhost:8080/v1/chat/completions \ [`llama-index-llms-ferrolabsai 0.0.1`](https://pypi.org/project/llama-index-llms-ferrolabsai/) is a reserved PyPI placeholder. The `0.1.0` release will ship a real `FerroLabsAI` LLM class implementing LlamaIndex's `LLM` interface (`complete`, `chat`, `stream_chat`, `achat`), with `trace_id`, `provider`, and `cost_usd` surfaced on every response. It will later be upstreamed into [`run-llama/llama_index`](https://github.com/run-llama/llama_index) for LlamaIndex hub discoverability. -Track progress: [Ferro AI Gateway roadmap](https://github.com/ferro-labs/ai-gateway/blob/main/ROADMAP.md). +Track progress: [Ferro Labs AI Gateway roadmap](https://github.com/ferro-labs/ai-gateway/blob/main/ROADMAP.md). ## Runnable example diff --git a/docs/frameworks/vercel-ai-sdk.mdx b/docs/frameworks/vercel-ai-sdk.mdx index b7f9209..c154783 100644 --- a/docs/frameworks/vercel-ai-sdk.mdx +++ b/docs/frameworks/vercel-ai-sdk.mdx @@ -1,6 +1,6 @@ --- title: Vercel AI SDK -description: Use Ferro Labs AI Gateway with the Vercel AI SDK as a drop-in via the OpenAI provider — route streamText, generateText, generateObject, and tool calls across 30+ providers from one URL. +description: "Use the Vercel AI SDK with Ferro Labs AI Gateway via the OpenAI provider — streamText, generateText, generateObject, and tool calls across 30 providers." keywords: [Vercel AI SDK gateway, Vercel AI SDK proxy, Vercel AI SDK multi-provider, AI SDK Anthropic, AI SDK Gemini, Next.js AI gateway, AI gateway streaming, OpenRouter alternative] sidebar_position: 5 --- @@ -145,4 +145,4 @@ Look for `x-trace-id` in the response headers and SSE chunks in the body. - [TypeScript SDK quickstart](/integrations/overview) — the underlying `@ferro-labs-ai/sdk` client - [Mastra](/frameworks/mastra) — multi-provider workflows built on the AI SDK - [LangChain.js](/frameworks/langchain-js) — for LangChain.js / LangGraph.js workloads -- [Routing policies](/guides/routing-policies) — what the gateway does behind the URL +- [Routing policies](/routing) — what the gateway does behind the URL diff --git a/docs/getting-started/architecture.mdx b/docs/getting-started/architecture.mdx index 3c5c5d2..b7b2392 100644 --- a/docs/getting-started/architecture.mdx +++ b/docs/getting-started/architecture.mdx @@ -1,77 +1,232 @@ --- title: Architecture -description: Reference architecture for deploying the Ferro Labs AI Gateway in production — component diagrams, traffic flow, scaling boundaries, and multi-provider routing design. -keywords: [AI gateway architecture, LLM proxy design, AI control plane, multi-provider routing, AI infrastructure, gateway deployment architecture] +description: "How the AI Gateway routes every surface through one routeTargets pipeline, plus the embedded dashboard, MCP subsystem, failover rules, and scaling boundaries." +keywords: [AI gateway architecture, routeTargets pipeline, circuit breaker failover, MCP subsystem, admin control plane, AI gateway scaling] --- -This page gives a practical architecture view similar to modern AI gateway docs: control points, traffic flow, and scaling boundaries. +This page describes how a single gateway instance is actually built: one routing +pipeline shared by every surface, the plugin stages that wrap it, the embedded +dashboard and admin control plane, the MCP subsystem, and the state that does +(and doesn't) survive a restart or scale across replicas. ## High-level architecture ```mermaid -flowchart LR - A[Client Apps\nSDKs / Services] --> B[AI Gateway] - B --> C[Routing Engine] - B --> D[Policy Layer\nAuth / Rate Limits / Guardrails] - B --> E[Observability\nLogs / Metrics / Traces] - C --> F[Provider: OpenAI] - C --> G[Provider: Anthropic] - C --> H[Provider: Gemini] - C --> I[Provider: Others] +flowchart TB + Client["Client SDK / App
(OpenAI-compatible)"] -->|Bearer token| Router["HTTP Router"] + + Router --> Admit["admitModel
pre-flight: does ANY target serve this model?"] + Admit -->|"no capable target"| Deny404["404 model_not_found
(on_error still runs)"] + Admit --> Before["before_request plugins
guardrails · rate-limit · budget · response-cache"] + Before -->|Reject| DenyPlugin["4xx / 429 / 402"] + Before --> Strategy["Strategy.SelectTargets
8 modes — decides ORDER only"] + + Strategy --> Pipeline["routeTargets (gateway_pipeline.go)
ONE walk for chat / streaming / embeddings / images"] + Pipeline --> Retry["per-target retry
(targets[].retry, every mode)"] + Retry --> CB{"circuit breaker
open?"} + CB -->|open, sibling healthy| Retry + CB -->|closed / half-open| Limiter["concurrency limiter
(targets[].concurrency)"] + Limiter --> Provider[("Provider API")] + Provider --> After["after_request plugins
(cache write, cost, logging)"] + After --> Client + CB -->|"all candidates open"| Deny503["503 upstream_unavailable"] + + Pipeline -.->|agentic tool calls| MCP + Router -.-> Admin + + subgraph MCP["MCP subsystem"] + direction LR + HTTPT["HTTP servers
(Streamable HTTP)"] + StdioT["stdio subprocesses
(no inherited env)"] + end + + subgraph Admin["Embedded dashboard + admin control plane"] + direction LR + Keys["API keys / sessions"] + Cfg["Config history / rollback"] + Logs["Request logs"] + end ``` +The dashboard is not a separate service — it's a static SPA bundle embedded in +the same binary and served from the gateway's root path, behind the same +bearer-auth chain as the admin API. + ## Request path ```mermaid sequenceDiagram participant App as Application - participant GW as AI Gateway - participant Policy as Policy Layer - participant Route as Routing Engine - participant Provider as Provider API - - App->>GW: OpenAI-compatible request - GW->>Policy: Validate auth + limits + plugins - Policy-->>GW: Allow / Reject - GW->>Route: Select provider strategy - Route-->>GW: Primary target + fallback plan - GW->>Provider: Forward normalized request - Provider-->>GW: Response / Error - GW-->>App: Final response + participant GW as Gateway (HTTP router) + participant Plug as Plugin stages + participant Pipe as routeTargets + participant Prov as Provider + + App->>GW: POST /v1/chat/completions (bearer token) + GW->>GW: admitModel — does any target serve this model? + GW->>Plug: before_request (guardrails, rate-limit, budget, cache) + Plug-->>GW: allow / Reject + GW->>Pipe: Strategy.SelectTargets(req) -> ordered keys + loop per candidate target + Pipe->>Pipe: skip if circuit OPEN + Pipe->>Prov: call under retry + breaker + concurrency limit + Prov-->>Pipe: response / error + alt pool mode (fallback, loadbalance, least-latency,
cost-optimized, ab-test) and target failed + Pipe->>Pipe: advance to next candidate + else named mode (single, conditional, content-based)
or success + Pipe->>Pipe: stop + end + end + Pipe-->>GW: response + target key (or error) + GW->>Plug: after_request (or on_error) + GW-->>App: final response ``` ## Core components +### Unified routing pipeline (`routeTargets`) + +This is the defining piece of the current architecture. Chat, streaming chat, +embeddings, and image generation all route through the **same** function, +`routeTargets` in `gateway_pipeline.go` — not four parallel implementations. +The strategy contributes target *order* and nothing else; everything that +happens once a target is chosen lives in the pipeline, once, for all four +surfaces: + +- per-target **retry** (`targets[].retry` — honoured under **every** routing + mode, not only `fallback`) +- the per-target **circuit breaker** (one breaker per `virtual_key`, shared + across all four surfaces — a target that only ever fails `/v1/embeddings` + can still trip the breaker that stops it serving chat) +- the per-target **concurrency limiter** (`targets[].concurrency` — + `max_concurrency` + `queue_size`; overflow returns 429) +- error classification, latency recording, metrics, and request logging + +Plugin stages run **outside** this walk on purpose: a retry that re-ran a +budget or guardrail plugin would bill or check the same request multiple +times for one call. The pass-through `/v1/*` proxy and the priced +`/v1/responses` surface both run through the same admission, plugin, and +routing lifecycle as the four routed surfaces — they aren't a separate, +ungoverned code path. + ### API compatibility layer -- Accepts OpenAI-compatible request/response formats. -- Lets application code remain stable while backend models/providers change. +Accepts OpenAI-compatible request/response shapes so application code stays +stable as backend models and providers change. Requests to endpoints the +gateway doesn't natively model (most non-chat OpenAI routes) fall through to +the pass-through proxy, which still runs the governed lifecycle above. + +### Routing strategy + +Selects and orders candidate targets — nothing more. All 8 modes implement +one method, `SelectTargets`: `single`, `fallback`, `loadbalance`, +`conditional`, `content-based`, `least-latency`, `cost-optimized`, and +`ab-test`. See [Routing](/routing) for per-strategy config and examples. + +### Plugin stages + +Global middleware (`before_request`, `after_request`, `on_error`) wrapping +the pipeline — guardrails, rate limiting, budget enforcement, response +caching, and request logging. See [Plugins](/plugins) for stage semantics and +failure policy. -### Routing engine +### MCP subsystem -- Supports all 8 strategies: `single`, `fallback`, `loadbalance`, `conditional`, `content-based`, `least-latency`, `cost-optimized`, and `ab-test`. -- Separates model selection policy from app logic. -- See [Routing policies](/guides/routing-policies) for YAML examples of each strategy. +External tool servers for agentic tool-calling, wired from `mcp_servers[]` in +config. Each entry is exactly one transport: `url` for Streamable HTTP, or +`command`/`args` for a stdio subprocess (which inherits **no** gateway +environment — only `PATH`/`HOME`/`LANG`/`TMPDIR` plus its own `env` block). +Guardrails and budget checks re-run on every agentic loop turn, not just the +first. See [MCP](/guides/mcp). -### Policy and controls +### Embedded dashboard + admin control plane -- Handles authentication, rate limiting, and plugin-level checks. -- Applies policy consistently for every provider. +The operations dashboard is a React SPA embedded in the OSS binary and served +from the gateway's root path — there's no standalone dashboard container or +second origin. The admin control plane behind it (API keys, dashboard +sessions, config history/rollback, request logs, audit trail) uses the same +bearer-auth chain the data plane does, so the in-browser Playground can call +`/v1/*` with the operator's own session. ### Observability -- Emits logs for every request and upstream outcome. -- Exposes metrics for latency, error rate, and provider health. +Emits Prometheus metrics at `/metrics` and, when configured, OpenTelemetry +traces. Health is split three ways: + +| Endpoint | Answers | Auth | +|---|---|---| +| `/livez` | Is the process up? | none | +| `/readyz` | Can it serve traffic? 503 `no routable targets` when zero targets are routable | none | +| `/health` | Deep diagnostic — per-provider status and circuit state | none | + +## Failover: pool vs named modes + +A circuit breaker doesn't produce failover by itself — a deployment with no +breaker configured still fails over. The two do different jobs: the breaker +makes failover **cheap** (skips a dead target's connection timeout), the +routing mode makes it **happen** (decides whether to try a sibling at all). + +| | Modes | On a target failure | +|---|---|---| +| **Pool** | `fallback`, `loadbalance`, `least-latency`, `cost-optimized`, `ab-test` | the walk advances to the next candidate | +| **Named** | `single`, `conditional`, `content-based` | the walk stops and reports the failure | + +A pool mode picks its head target for a reason that's about the *pool* — +spread load, take the cheapest, take the fastest — from targets declared +interchangeable, so handing the request to a sibling is what was configured. +A named mode picks its head because something named that target +specifically; serving from elsewhere would demote the rule to a suggestion. + +**Every mode skips a target whose circuit is open** — that rule is +independent of pool vs named. When another configured target serves the +model, the open one is silently passed over, even under `single`, +`conditional`, or `content-based` where an operator named one target on +purpose. When *no* other target serves the model, the walk still attempts +the open one (so a model that plainly exists never 404s), the breaker +refuses it, and the caller gets `503 upstream_unavailable` instead. + +## Scaling boundaries + +Several pieces of gateway state are **per-process** and do not share across +replicas unless noted: + +- Circuit breaker state, the least-latency tracker's p50 samples, and the + `rate-limit` plugin's token buckets all reset with the process and are not + shared across instances. +- The `response-cache` and `budget` plugins are in-memory by default; budget + spend does not survive a restart and is not a durable billing ledger. +- The per-IP HTTP rate limiter (`RATE_LIMIT_RPS`) is also per-process. + +What **does** persist and can be shared, when backed by `sqlite` or +`postgres` (`API_KEY_STORE_BACKEND`, `CONFIG_STORE_BACKEND`, +`REQUEST_LOG_STORE_BACKEND`): API keys, dashboard sessions, config history, +the audit trail, and request logs. Point every replica at the same backend +DSN to share those across a fleet; the in-memory default (used when these are +unset) keeps each replica's copy independent and loses it on restart. ## Deployment recommendations -- Start with one gateway instance behind a reverse proxy. -- Move to multiple gateway replicas with shared configuration for HA. -- Add provider-level fallback before introducing weighted distribution. -- Use metrics + request logs to tune routing rules over time. +- Start with one gateway instance; `ferrogw init` scaffolds a config from + whichever provider credentials are already in the environment. +- Set `GATEWAY_ENV=production` before exposing an instance publicly — it + refuses to start with `ALLOW_UNAUTHENTICATED_PROXY=true` or a wildcard + `CORS_ORIGINS`, and warns on the in-memory key store. +- Move to multiple replicas behind a load balancer for HA; point + `API_KEY_STORE_BACKEND`/`CONFIG_STORE_BACKEND`/`REQUEST_LOG_STORE_BACKEND` + at a shared `postgres` DSN so keys, config history, and logs agree across + instances (per-process state above still won't share). +- Configure a `circuit_breaker` on any target that can fail, and prefer a + pool routing mode (`fallback`/`loadbalance`) over a named one where + interchangeable providers exist — that's what makes failover automatic + rather than a manual failure to notice. +- Use `/metrics` and `/admin/logs` to tune retry, concurrency, and routing + weights over time. ## Related pages - [Request lifecycle](/getting-started/request-lifecycle) -- [Routing policies](/guides/routing-policies) -- [Observability](/guides/observability) +- [Routing](/routing) +- [Plugins](/plugins) +- [MCP](/guides/mcp) +- [Server settings](/operations/server-settings) +- [Monitoring](/operations/monitoring) diff --git a/docs/getting-started/concepts.mdx b/docs/getting-started/concepts.mdx index e85ce8a..3822d8e 100644 --- a/docs/getting-started/concepts.mdx +++ b/docs/getting-started/concepts.mdx @@ -1,56 +1,84 @@ --- title: Concepts -description: "Core concepts of the Ferro Labs AI Gateway — OpenAI-compatible routing across 30 providers, 8 strategies, plugin pipeline, MCP tool-calling, and the difference between routes, targets, and strategies." -keywords: [AI gateway concepts, LLM routing strategies, AI plugins, MCP tool calling, OpenAI wire format, gateway routing, content-based routing, ab-test routing] +description: "Core concepts of the Ferro Labs AI Gateway: the targets allowlist, 8 routing strategies, the plugin pipeline, MCP tool-calling, and the embedded dashboard." +keywords: [AI gateway concepts, LLM routing strategies, AI plugins, MCP tool calling, OpenAI wire format, capability matrix, plugin pipeline, targets allowlist] --- +The gateway sits between your application and 30 LLM providers, speaking one OpenAI-compatible wire format on the way in and translating to each provider's native API on the way out. This page defines the vocabulary the rest of the docs use: providers, targets, strategies, plugins, and MCP servers. + ## OpenAI-compatible API The gateway speaks the OpenAI wire format for chat completions, embeddings, images, and model listing. Any client that works with OpenAI will work with the gateway after changing only the `base_url`. Provider credentials, model routing, and policy enforcement happen inside the gateway — your application code is unaffected. +## Providers and targets + +A **provider** is a registered AI API backend (e.g., OpenAI, Anthropic, Bedrock). The gateway supports [30 providers](/providers). A provider is *registered* when its required environment variable(s) are set — no code changes needed. + +A **target** is a config entry (`targets[]`) that references a registered provider by `virtual_key`. `targets` is an **allowlist on every routed surface**: setting a provider's env var only registers it — the provider serves requests only if it also appears in `targets[]`. A model owned solely by a provider that is registered but not listed under `targets` returns `404 model_not_found`. + +Each target carries: + +| Field | Purpose | +|---|---| +| `virtual_key` | Names the registered provider this target routes to (required) | +| `weight` | Relative share under `loadbalance`; `0` drains the target (ignored by every other mode) | +| `retry` | `attempts`, `on_status_codes`, `initial_backoff_ms` — honoured under **every** routing mode, not just `fallback` | +| `circuit_breaker` | `failure_threshold`, `success_threshold`, `timeout` — one breaker per target, shared across chat, streaming, embeddings, and images | +| `concurrency` | `max_concurrency`, `queue_size` — caps in-flight requests per target; overflow returns `429` | +| `models` | Operator-declared model IDs this target serves, additive to the model catalog and live discovery; exact IDs only, no wildcards | + ## Routing strategies -The strategy controls which provider target receives each request. Configure it with the `strategy.mode` key in your config file. +The strategy controls which target(s) a request is offered to, and in what order. Configure it with `strategy.mode`. | Strategy | Description | |---|---| | `single` | Always route to the first target. Simplest setup. | -| `fallback` | Try targets in order; retry the next on failure with exponential backoff. | +| `fallback` | Try targets in declared order; advance to the next target on failure. | | `loadbalance` | Distribute requests across targets by weight. | -| `conditional` | Evaluate rules (model name, model prefix) to pick a target per request. | -| `least-latency` | Route to the target with the lowest P50 latency, using a rolling tracker. | +| `conditional` | Match `model` (exact) or `model_prefix` (prefix) against declared rules; first match wins. | +| `least-latency` | Route to the compatible target with the lowest observed p50 latency. | | `cost-optimized` | Estimate input cost from the model catalog and route to the cheapest compatible target. | -| `content-based` | Route based on prompt content using substring match or regex. First rule match wins. | +| `content-based` | Match user-message content by substring or regex; first rule match wins. | | `ab-test` | Split traffic across labeled variants by weight for comparison testing. | -See [Routing policies](/guides/routing-policies) for YAML examples of each strategy. +One pipeline governs chat, streaming, embeddings, and images, so retry and circuit-breaking behave identically across all four. Strategies split into two families: -## Providers and targets +- **Pool modes** (`fallback`, `loadbalance`, `least-latency`, `cost-optimized`, `ab-test`) advance past a failed target to the next candidate. +- **Named modes** (`single`, `conditional`, `content-based`) commit to their chosen target and report its failure rather than trying another. -A **provider** is a registered AI API backend (e.g., OpenAI, Anthropic). The gateway supports [30 providers](/guides/providers). Providers are enabled by setting environment variables — no code needed. +Every mode skips a target whose circuit is open; if every candidate's circuit is open, the request is still attempted and returns `503`. -A **target** is a reference to a provider in your config. Targets carry optional fields: `weight` (for load balancing), `retry` (attempts + status codes), and `circuit_breaker` (thresholds + timeout). +See [Routing](/routing) for per-strategy configuration and YAML examples. ## Model aliases Aliases map short names to full model IDs. They are resolved before routing, so `cheap` can map to `gemini-1.5-flash` and every request to `model: cheap` is transparently sent to Gemini. +## Capability matrix + +`GET /v1/capabilities` reports, per provider, which OpenAI chat parameters it can express: `forward` (sent as-is), `translate` (mapped to an equivalent), or `unsupported`. When a request sets a parameter a routed provider can't express, `compatibility.on_unsupported_param` decides what happens: `warn` (default — forward it anyway and log), `drop` (remove it and log), or `reject` (`400` naming the parameter). + ## Plugins -Plugins run at three lifecycle stages: `before_request`, `after_request`, and `on_error`. +Plugins are **global**, configured under a single top-level `plugins:` list — there is no per-route or per-target plugin field. Each entry runs at one of three stages: `before_request`, `after_request`, or `on_error`. ### OSS plugins -These 6 plugins ship with the open-source gateway: +These 6 plugins ship with the open-source gateway. Three of them are multi-stage — they must be listed once per stage with **byte-identical config**, or the gateway refuses to start: -| Plugin | Stage | Purpose | +| Plugin | Stage(s) | Purpose | |---|---|---| -| `word-filter` | before_request | Block requests containing banned words | -| `max-token` | before_request | Enforce token and message count limits | -| `response-cache` | before_request | Cache exact-match responses in memory | -| `request-logger` | before_request | Emit structured logs, optionally persist to SQLite/Postgres | -| `rate-limit` | before_request | Token-bucket rate limiting (global, per-key, per-user) | -| `budget` | before_request + after_request | Track and enforce per-key spend limits | +| `word-filter` | `before_request` (optionally also `after_request`) | Reject request or response text containing a blocked substring | +| `max-token` | `before_request` | Reject requests exceeding token, message-count, or input-length limits | +| `rate-limit` | `before_request` | Token-bucket rate limiting (global, per-key, per-user) | +| `budget` | `before_request` **+** `after_request` | Check and record per-API-key USD spend; over the cap returns `402 insufficient_quota` | +| `response-cache` | `before_request` **+** `after_request` | Serve identical repeated chat requests from an in-memory cache | +| `request-logger` | `before_request` **+** `after_request` **+** `on_error` | Emit structured logs, optionally persist to SQLite/Postgres | + +**Failure policy**: a plugin's `Reject` verdict is always honoured — it becomes a `4xx`, `429` (rate-limit), or `402` (budget) response. A plugin *error* (the plugin itself broke, not a deliberate rejection) fails **closed** (`500`) for guardrail, auth, ratelimit, and transform plugins, but fails **open** (logged, request continues) for logging and metrics plugins. + +**SkipProvider**: when `response-cache` serves a hit, it sets `SkipProvider` on the request, which skips only the call to the upstream provider — every remaining `before_request` plugin and the whole `after_request` stage still run, so rate limiting, budget checks, and logging behave the same whether or not the response came from cache. ### Ferro Labs Managed plugins @@ -68,24 +96,38 @@ These 5 plugins require [Ferro Labs Managed](https://www.ferrolabs.ai/) because The 5 enterprise plugins require a Ferro Labs Managed account. [Join the waitlist →](https://www.ferrolabs.ai/) ::: -See [Plugins](/guides/plugins) for full configuration examples. +See [Plugins](/plugins) for full configuration examples. ## MCP integration -Model Context Protocol (MCP) lets you connect external tool servers to the gateway. When `mcp_servers` are configured, the gateway injects available tools into every chat completion request and runs an agentic loop when the model returns `tool_calls`. This works transparently — clients receive the final text response without needing to implement the tool loop themselves. +Model Context Protocol (MCP) lets you connect external tool servers to the gateway. Each `mcp_servers[]` entry sets exactly one of `url` (Streamable HTTP) or `command` (+`args`, a stdio subprocess); a stdio subprocess inherits **no** gateway environment — only `PATH`/`HOME`/`LANG`/`TMPDIR` plus whatever you list explicitly under `env`. + +Tool injection is conditional: the gateway injects available MCP tools into a chat completion **only when the request carries no `tools` array of its own**. A request that already supplies its own tools passes through untouched — the gateway does not merge or de-duplicate. When the model responds with `tool_calls` the gateway owns, it calls the tool over MCP, appends the result as a `tool` message, and re-sends to the model — up to a bounded depth — invisibly to the caller. Guardrails and budget checks run on every loop turn, not just the first. -MCP Phase 1 (tool injection + agentic loop) shipped in v0.8.0. Streaming support for MCP requests was added in v1.0.0. +Setting `required: true` on a server gates `GET /readyz` on that server's initialize handshake — if it isn't ready, the instance stops serving traffic entirely, including requests that need no tools. See [MCP integration](/guides/mcp) for setup and examples. +## Dashboard + +The operations dashboard ships **embedded** in the OSS binary and is served from the gateway's own root — there's no standalone dashboard container or separate image to deploy. It surfaces request logs, API key management, configured plugins, and config history. + ## Ferro Labs Managed -[Ferro Labs Managed](/ferrocloud/overview) is the managed, multi-tenant version of the AI Gateway hosted by Ferro Labs. It wraps the same OSS engine with per-tenant isolation, a management dashboard, durable billing, [semantic caching](/ferrocloud/semantic-cache), SSO/SAML, audit logs, and the 5 enterprise security plugins listed above. See [OSS vs Ferro Labs Managed](/guides/oss-vs-ferrocloud) for a full comparison. +[Ferro Labs Managed](/ferrocloud/overview) is the managed, multi-tenant version of the AI Gateway hosted by Ferro Labs. It wraps the same OSS engine with per-tenant isolation, durable billing, [semantic caching](/ferrocloud/semantic-cache), SSO/SAML, audit logs, and the 5 enterprise security plugins listed above. See [OSS vs Ferro Labs Managed](/guides/oss-vs-ferrocloud) for a full comparison. ## Observability -- **Prometheus metrics** — scraped at `/metrics`. Includes request counts, latency histograms, token usage, and cache hit rates. -- **Structured JSON logs** — emitted to stdout with a per-request `trace_id` for log correlation. -- **Health endpoint** — `GET /health` returns per-provider availability with latency measurements. +- **Prometheus metrics** — scraped at `/metrics` (requires a bearer token with `read_only` or `admin` scope). Includes request counts, latency histograms, token usage, and cache hit rates. +- **Structured JSON logs** — emitted to stdout, correlated by the `X-Request-ID` trace ID also returned on every response. +- **Health checks are split three ways**: `GET /livez` reports the process is alive; `GET /readyz` reports whether at least one target is routable (`503`, reason `no routable targets`, when none is); `GET /health` returns a deeper diagnostic — per-provider registration and circuit-breaker state. + +See [Monitoring](/operations/monitoring) for details. + +## Related -See [Observability](/guides/observability) and [Monitoring](/operations/monitoring) for details. +- [Architecture](/getting-started/architecture) +- [Request lifecycle](/getting-started/request-lifecycle) +- [Configuration](/getting-started/configuration) +- [Routing](/routing) +- [Plugins](/plugins) diff --git a/docs/getting-started/configuration.mdx b/docs/getting-started/configuration.mdx index 46eadc2..c509fc4 100644 --- a/docs/getting-started/configuration.mdx +++ b/docs/getting-started/configuration.mdx @@ -1,7 +1,7 @@ --- title: Configuration -description: "Complete v1.0.0 configuration reference for the Ferro Labs AI Gateway — routes, targets, 8 routing strategies, model aliases, all 6 built-in plugins, MCP servers, and budget controls." -keywords: [AI gateway configuration, config.yaml, LLM routing config, model aliases, YAML configuration, gateway config reference, content-based routing config, ab-test config] +description: "Complete v1.4.x config reference for the Ferro Labs AI Gateway — targets, retry, circuit breakers, all 8 routing strategies, plugins, and MCP servers." +keywords: [AI gateway configuration, config.yaml, LLM routing config, targets concurrency, compatibility on_unsupported_param, MCP servers config, observability tracing config, budget plugin config] --- import Head from '@docusaurus/Head'; @@ -11,13 +11,13 @@ import Head from '@docusaurus/Head'; "@context": "https://schema.org", "@type": "TechArticle", "headline": "Ferro Labs AI Gateway Configuration Reference", - "description": "Complete v1.0.0 configuration reference for the Ferro Labs AI Gateway — routes, targets, 8 routing strategies, model aliases, all 6 built-in plugins, MCP servers, and budget controls.", + "description": "Complete v1.4.x config reference for the Ferro Labs AI Gateway — targets, retry, circuit breakers, all 8 routing strategies, plugins, and MCP servers.", "url": "https://docs.ferrolabs.ai/getting-started/configuration/" })} -:::info v1.0.0 reference -This configuration reference covers Ferro Labs AI Gateway v1.0.0 — the first stable release with semver guarantees. All configuration keys documented here are part of the stable API. +:::info As of v1.4.x +This page documents the config schema shipped by v1.4.x. `apiVersion` is advisory — an unrecognized value is kept and only logged as a warning, so a newer config file still starts on an older binary. ::: The gateway loads configuration from a YAML or JSON file at the path set by `GATEWAY_CONFIG`. @@ -27,7 +27,24 @@ export GATEWAY_CONFIG=./config.yaml ./ferrogw ``` -Supported extensions: `.yaml`, `.yml`, `.json`. +Supported extensions: `.yaml`, `.yml`, `.json`. Decoding is **strict** — an unknown key is rejected (with its name and line number) rather than silently ignored, and the same strict decoder validates `PUT`/`POST /admin/config`, so a config pushed through the admin API can't disagree with a file on what's valid. Loading only decodes; a separate validation pass (`ValidateConfig`, also run by `ferrogw validate`) checks the values — an unroutable `target_key`, a negative weight, a duplicate multi-stage plugin — and a failure on either exits the process (`os.Exit(1)`). + +## Top-level fields + +| Key | Type | Default | Description | +|---|---|---|---| +| `apiVersion` | string | `v1` | Advisory schema version. Never causes a load failure. | +| `max_request_bytes` | int64 | `10485760` (10 MiB) | Body-size cap for `/v1/*` and admin write endpoints. Over the limit = HTTP 413. Batch/file uploads are exempt. | +| `request_timeout` | Go duration string | unset (no deadline) | Bounds one non-streaming request end to end — plugin stages, provider call, and every retry/fallback attempt. Streaming requests are exempt, **except** an MCP agentic loop, which is delivered as a single chunk and is treated like any other non-streaming request. | +| `strategy` | object | mode `single` | Routing configuration — see [Strategy](#strategy). | +| `targets` | array | — (at least 1 required) | Provider targets — see [Targets](#targets). | +| `batch_target` | string | unset (surface returns 501) | `virtual_key` of the target that serves `/v1/files*` and `/v1/batches*`. Must name a configured target whose provider supports batch pass-through (`openai`, `azure-openai`, `groq`, `novita`, `qwen`). | +| `responses_target` | string | unset (id sub-routes return 501) | `virtual_key` of the target that serves the stateful `/v1/responses/{id}` sub-routes (retrieve/delete/cancel/input_items). `POST /v1/responses` (create) still routes by model regardless. | +| `aliases` | map | — | Friendly model name → concrete model id, resolved before routing and plugins. | +| `plugins` | array | — | Plugin middleware entries — see [Plugins](#plugins). | +| `mcp_servers` | array | — | External MCP tool servers — see [MCP servers](#mcp-servers). | +| `compatibility` | object | `on_unsupported_param: warn` | See [Compatibility](#compatibility). | +| `observability` | object | NoOp (tracing off) | See [Observability](#observability). | ## Strategy @@ -38,16 +55,22 @@ strategy: mode: fallback # single | fallback | loadbalance | conditional | least-latency | cost-optimized | content-based | ab-test ``` -| Mode | Description | -|---|---| -| `single` | Route every request to the first target. | -| `fallback` | Try targets in order; retry on failure with exponential backoff. | -| `loadbalance` | Weighted random distribution across targets. | -| `conditional` | Rule-based: match a request field (e.g. model name) to a target. | -| `least-latency` | Route to the target with the lowest P50 latency (rolling tracker). | -| `cost-optimized` | Use the built-in model catalog to estimate prompt cost and pick the cheapest compatible target. | -| `content-based` | Route based on user message content using substring or regex matching. | -| `ab-test` | Split traffic across labeled variants by weight for comparison testing. | +| Mode | Family | Description | +|---|---|---| +| `single` | Named | Route every request to `targets[0]` only. | +| `fallback` | Pool | Try targets in declared order; advance to the next on failure. | +| `loadbalance` | Pool | Weighted random distribution across targets (`targets[].weight`). | +| `conditional` | Named | Match a request field (`model` or `model_prefix`) to a target; first match wins. | +| `least-latency` | Pool | Route to the compatible target with the lowest observed p50 wall-clock latency. | +| `cost-optimized` | Pool | Estimate prompt cost from the model catalog and pick the cheapest compatible target. | +| `content-based` | Named | Route on user-message content by substring or regex; first match wins. | +| `ab-test` | Pool | Split traffic across labeled, weighted variants for comparison testing. | + +The mode families matter for failure handling: a **pool** mode (`fallback`, `loadbalance`, `least-latency`, `cost-optimized`, `ab-test`) advances the request pipeline to the next candidate target when one fails. A **named** mode (`single`, `conditional`, `content-based`) commits to the target it picked and reports that target's failure — it does not fail over to a sibling target just because one exists. + +Two rules hold under **every** mode, named or pool: +- `targets[].retry` is honored regardless of mode — it re-asks the *same* target the number of configured times. Whether a *different* target is asked afterward is the mode's decision alone. +- A target whose circuit breaker is open is skipped when the routable set is chosen. When every candidate target's circuit is open, the request is still attempted and answered `503` — that's a different signal from `404 model_not_found` ("nothing serves this model" vs. "everything that could serve it is currently down"). ### Conditional rules @@ -56,17 +79,19 @@ strategy: mode: conditional conditions: - key: model - value: gpt-4o-mini + value: gpt-4o target_key: openai - - key: model - value: claude-3-5-sonnet-20241022 + - key: model_prefix + value: claude- target_key: anthropic - - key: model - value: gemini-1.5-flash - target_key: gemini +targets: + - virtual_key: openai # targets[0] doubles as the no-match fallback + - virtual_key: anthropic ``` -Rules are evaluated in order. `key` names the request field to inspect (e.g. `model`), `value` is the exact value it must match, and `target_key` is the target to route to. +`key` is one of `model` (exact match) or `model_prefix` (`strings.HasPrefix` match) — a closed set validated at load; anything else is a config error, not a silent no-op. `value` is what `key` is matched against, and `target_key` must name a configured `targets[].virtual_key`. + +Rules are evaluated in order; the first match wins and the request commits to that target — a request for a model the matched target doesn't actually serve is `404 model_not_found`, even when another configured target does serve it. Write another rule rather than relying on failover; `conditional` is a named mode. Unmatched requests fall to `targets[0]`. ### Content-based routing @@ -74,18 +99,24 @@ Rules are evaluated in order. `key` names the request field to inspect (e.g. `mo strategy: mode: content-based content_conditions: + - type: prompt_regex + value: "(?i)\\b(code|function|class|implement)\\b" + target_key: deepseek - type: prompt_contains value: "translate" - target_key: deepl-provider - - type: prompt_regex - value: "(?i)(code|function|class|def |import )" + target_key: gemini + - type: prompt_not_contains + value: "confidential" target_key: openai - - type: prompt_contains - value: "summarize" - target_key: anthropic +targets: + - virtual_key: openai # no-match fallback + - virtual_key: deepseek + - virtual_key: gemini ``` -Three condition types: `prompt_contains` (case-insensitive substring), `prompt_not_contains`, and `prompt_regex` (Go regexp). Regex patterns are compiled at startup — invalid patterns cause a startup error. First match wins; unmatched requests fall to the first target. +Three condition types, evaluated over **user-role messages only** (system and assistant content is never inspected): `prompt_contains` (case-insensitive substring), `prompt_not_contains` (true when no user message contains the value — matches broadly, so ordering matters), and `prompt_regex` (Go regexp, compiled at startup — an invalid pattern is a startup error). First match wins and the request commits to that target; unmatched requests fall to `targets[0]`. + +`content-based` is a named mode, so a failure at the matched target is the request's answer, not a trigger to try another target. ### A/B test routing @@ -94,57 +125,82 @@ strategy: mode: ab-test ab_variants: - target_key: openai - weight: 80 + weight: 70 label: control - target_key: anthropic - weight: 20 + weight: 30 label: challenger ``` -Weights are relative. Each variant carries a `label` field, which is currently emitted only to the gateway's DEBUG logs (not to events or a queryable log column). +Weights are relative (`weight / sum(weights)`); a zero weight drains a variant to no traffic, a negative weight or an all-zero set is a load error. The draw is over **eligible** variants only — a variant whose provider doesn't serve the requested model never wins that draw. Each variant's `label` is logged with every routed request for analytics. `ab-test` is a pool mode: a failure at the drawn variant advances to the next eligible one. ## Targets -Targets are provider references. Each `virtual_key` must match a registered provider (see [Provider configuration](/guides/providers-config)). +Targets are provider references. Each `virtual_key` must name a registered provider (registration comes from setting that provider's env vars — see [Provider configuration](/providers/configuration)). **`targets` is an allowlist on every routed surface**: a provider can be fully registered via its env vars and still serve nothing if it isn't listed here — a model no configured target owns answers `404 model_not_found`. ```yaml targets: - virtual_key: openai - weight: 70 # used by loadbalance strategy + weight: 1.0 # relative share under loadbalance only; ignored elsewhere retry: attempts: 3 - on_status_codes: [429, 502, 503] - initial_backoff_ms: 100 + on_status_codes: [429, 502, 503] # omit for the default policy: transport errors + 408/429/5xx + initial_backoff_ms: 100 # base for full-jitter exponential backoff circuit_breaker: - failure_threshold: 5 # failures before opening - success_threshold: 2 # successes before closing - timeout: "30s" # time in open state before half-open probe + failure_threshold: 5 # consecutive failures before opening (default 5) + success_threshold: 1 # half-open successes needed to close (default 1) + max_half_threshold: 1 # concurrent probes allowed while half-open (default 1) + timeout: 30s # time in open state before a half-open probe (default 30s) + concurrency: + max_concurrency: 32 # simultaneous in-flight requests to this target (1..10000) + queue_size: 500 # requests allowed to wait for a slot; overflow => HTTP 429 + + - virtual_key: gemini + models: # models this target serves, declared by the operator — + - gemini-2.5-flash # ADDITIVE to the model catalog and live discovery +``` - - virtual_key: anthropic - weight: 30 - retry: - attempts: 2 +| Field | Type | Notes | +|---|---|---| +| `virtual_key` | string, required | Names a registered provider. | +| `weight` | float64 | Relative share under `loadbalance` only. `0` drains the target. Negative or all-zero across the target set is a load error. | +| `models` | []string | Operator-declared model ids this target serves, in addition to whatever the catalog and live discovery already report. Exact ids only — wildcards are rejected at load. Purely additive: it never hides models the target already serves, and declaring one the catalog already knows is a harmless no-op. Advertised on `/v1/models`. | +| `retry` | object | `attempts` (int, `1` = no retry), `on_status_codes` ([]int), `initial_backoff_ms` (int, default 100). Applies under **every** routing mode — it re-asks this one target; it does not by itself try a different target. | +| `circuit_breaker` | object | One breaker per `virtual_key`, shared across chat, streaming, embeddings, *and* image generation to that target — a target that fails only on one surface still opens the shared breaker and stops serving all four. | +| `concurrency` | object | `max_concurrency` (1..10000) and `queue_size`. A streaming request holds its slot until the stream ends, not just until headers arrive. Overflow past the queue is `429`. | + +A `429`, a client disconnect, a caller-supplied deadline, an unsupported-parameter rejection, and a concurrency shed do **not** count toward opening the circuit breaker. A redirect, a `5xx`, a connection failure, and the gateway's own `request_timeout` or stream idle bound elapsing all do. + +### Declared models (`targets[].models`) {#declared-models} + +Reach for `models` when a target serves a model none of the automatic sources can see: an id newer than the catalog, a preview or regional name, a self-hosted deployment behind `_BASE_URL`, or a provider with no `/models` endpoint to enumerate. It's provider-agnostic — every target has the field — and strictly additive, so it can never accidentally unroute a model the target already serves. + +## Batch and Responses backends + +```yaml +batch_target: openai # serves /v1/files* and /v1/batches* pass-through +responses_target: openai # serves the stateful /v1/responses/{id} sub-routes ``` +Both endpoints carry no `model` — a batch references an opaque, provider-scoped id, so a single named target serves the whole surface with native ids and zero gateway state. `batch_target`'s provider must support batch pass-through (`openai`, `azure-openai`, `groq`, `novita`, `qwen`); `responses_target` is typically `openai` or `xai`. Omitting either leaves the corresponding surface returning `501` — `POST /v1/responses` (create) still routes normally by model even with `responses_target` unset. + ## Model aliases -Aliases resolve before routing. They let you use short names and switch backing models without changing client code. +Aliases resolve before routing and before plugins run. They let you use short names and swap the backing model without changing client code. ```yaml aliases: fast: gpt-4o-mini - smart: claude-3-5-sonnet-20241022 - cheap: gemini-1.5-flash + smart: claude-sonnet-4-6 + cheap: gemini-2.5-flash code: deepseek-coder ``` -A client that requests `model: cheap` will receive a response from Gemini 1.5 Flash. +No empty names or values, no self-reference, and no chained aliases (an alias pointing at another alias is rejected at load). ## Plugins -Each plugin entry specifies its `name`, `type`, `stage`, and `config` map. Set `enabled: false` to disable without removing the entry. - -### OSS plugins +Each entry specifies `name`, `type` (a label only — the plugin's own reported type decides fail-open vs. fail-closed, not this field), `stage`, `enabled`, and `config`. Order within a stage is execution order. ```yaml plugins: @@ -153,7 +209,7 @@ plugins: stage: before_request enabled: true config: - blocked_words: ["password", "confidential"] + blocked_words: ["password", "secret"] case_sensitive: false - name: max-token @@ -163,160 +219,215 @@ plugins: config: max_tokens: 4096 max_messages: 50 - - - name: response-cache - type: transform - stage: before_request - enabled: true - config: - max_age: 300 - max_entries: 1000 - - - name: request-logger - type: logging - stage: before_request - enabled: true - config: - level: info - persist: true - backend: sqlite - dsn: ferrogw-requests.db + max_input_length: 0 # 0 = no limit - name: rate-limit - type: ratelimit + type: guardrail stage: before_request enabled: true config: requests_per_second: 100 burst: 100 - key_rpm: 60 # per API key limit (v0.8.5+) - user_rpm: 30 # per user ID limit (v0.8.5+) - - - name: budget - type: guardrail - stage: before_request # also register at after_request to record costs - enabled: true - config: - spend_limit_usd: 10.00 - store_id: default # instances sharing store_id share spend data + key_rpm: 60 # optional per-API-key limit + user_rpm: 30 # optional per-user limit (keyed on Request.User) ``` -:::tip Budget plugin dual registration -The `budget` plugin should be registered twice — once at `before_request` (to reject over-limit keys) and once at `after_request` (to record token costs). Use the same `store_id` in both entries to share the spend counter. +:::danger Multi-stage plugins need byte-identical config +`response-cache`, `budget`, and `request-logger` each implement more than one stage — response-cache checks the cache before the provider call and stores after it, budget checks spend before and records after, request-logger writes a row at every stage it's listed at. **Every entry for the same plugin must carry identical config** (checked as JSON-encoded equality of the whole `config` block), or **the gateway refuses to start**. The same `store_id`/`name` alone is not enough — any other key differing between entries is still two disagreeing configs. +::: + ```yaml plugins: - - name: budget + # response-cache: before_request serves a cache hit (skips only the + # provider call — every other before_request plugin still runs); after_request stores it. + - name: response-cache + type: transform stage: before_request enabled: true config: - spend_limit_usd: 10.00 - store_id: default - - name: budget + max_age: 300 + max_entries: 1000 + - name: response-cache + type: transform stage: after_request enabled: true config: - store_id: default -``` -::: - -### Ferro Labs Managed plugins - -These plugins require a [Ferro Labs Managed](https://www.ferrolabs.ai/) account: + max_age: 300 + max_entries: 1000 -```yaml -plugins: - - name: pii-redact - type: guardrail + # request-logger: register at all three stages it implements, including + # on_error — a failed request never reaches after_request, so without the + # on_error entry a failure produces no terminal row at all. + - name: request-logger + type: logging stage: before_request enabled: true config: - action: redact # redact | block - redact_mode: replace_type - apply_to: input - - - name: secret-scan - type: guardrail - stage: before_request + level: info + persist: true + - name: request-logger + type: logging + stage: after_request enabled: true config: - action: block - - - name: prompt-shield - type: guardrail - stage: before_request + level: info + persist: true + - name: request-logger + type: logging + stage: on_error enabled: true config: - action: block - threshold: 0.90 - apply_to: user_messages + level: info + persist: true - - name: schema-guard + # budget: before_request checks accumulated spend against the limit; + # after_request records what the completed request cost. + - name: budget type: guardrail - stage: after_request + stage: before_request enabled: true config: - apply_to: output - action: block - extract_json: true - schema: - type: object - required: [name, confidence] - properties: - name: - type: string - confidence: - type: number - minimum: 0 - maximum: 1 - - - name: regex-guard + store_id: default # instances sharing store_id share spend counters + spend_limit_usd: 10.0 # 0 = unlimited + input_per_m_tokens: 3.0 # USD per 1M prompt tokens + output_per_m_tokens: 15.0 # USD per 1M completion tokens + cache_read_per_m_tokens: 0.30 # optional: USD per 1M cached-read prompt tokens + cache_write_per_m_tokens: 3.75 # optional: USD per 1M cache-write tokens + max_keys: 10000 # max tracked API keys; evicts lowest-spend key at cap + - name: budget type: guardrail - stage: before_request + stage: after_request enabled: true config: - action: block - patterns: - - "(?i)drop\\s+table" - - "(?i)delete\\s+from" + store_id: default + spend_limit_usd: 10.0 + input_per_m_tokens: 3.0 + output_per_m_tokens: 15.0 + cache_read_per_m_tokens: 0.30 + cache_write_per_m_tokens: 3.75 + max_keys: 10000 ``` -:::note Ferro Labs Managed feature -The 5 enterprise plugins (pii-redact, secret-scan, prompt-shield, schema-guard, regex-guard) require a Ferro Labs Managed account. [Join the waitlist →](https://www.ferrolabs.ai/) -::: +An exhausted budget rejects the request with **`402 insufficient_quota`**, not a 429 — explicitly so SDKs don't retry a request that will never succeed. `PromptTokens` from the provider is inclusive of `CacheReadTokens`, so setting `cache_read_per_m_tokens` bills the cached subset at that rate and the remainder at `input_per_m_tokens`; leaving it unset bills the whole prompt at `input_per_m_tokens` as before. + +Plugin `config` string values support `${VAR}` — see [`${VAR}` resolution](#var-resolution) below. -See [Plugins](/guides/plugins) for all 6 built-in plugins and their full config options. +See [Plugins](/plugins) for all 6 built-in OSS plugins (`word-filter`, `max-token`, `response-cache`, `request-logger`, `rate-limit`, `budget`) and their full option sets. The 5 additional guardrails (pii-redact, secret-scan, prompt-shield, schema-guard, regex-guard) are Ferro Labs Managed-only — see [Enterprise plugins](/plugins/enterprise). ## MCP servers -Configure Model Context Protocol tool servers for agentic tool-calling. Streaming requests are supported as of v1.0.0. +Configure external MCP tool servers for agentic tool-calling. When `mcp_servers` is configured, the gateway injects the discovered tools into a chat completion request **only when the request itself carries no `tools` of its own** — a caller-supplied `tools` array passes through untouched and MCP sits out entirely for that request. + +Each entry sets exactly one of `url` (Streamable HTTP transport) or `command` (+ `args`, stdio subprocess transport). ```yaml mcp_servers: + # Streamable HTTP transport - name: filesystem url: "http://localhost:3001/mcp" - timeout_seconds: 10 - max_call_depth: 3 + timeout_seconds: 10 # per tool call, both transports; default 30 + max_call_depth: 3 # agentic loop turn cap; min positive value across servers wins, default 5 - name: database url: "https://mcp-db.internal/mcp" headers: Authorization: "Bearer ${MCP_DB_TOKEN}" - allowed_tools: + allowed_tools: # empty = all discovered tools exposed - query_readonly - list_tables timeout_seconds: 15 max_call_depth: 5 + required: false # true gates /readyz on this server's initialize handshake + + # stdio transport — the gateway launches and owns the subprocess. + # It does NOT inherit the gateway's environment: only PATH/HOME/LANG/TMPDIR + # (when set) plus the explicit `env` map below reach the child process. + - name: brave-search + command: npx + args: + - -y + - "@modelcontextprotocol/server-brave-search" + env: + BRAVE_API_KEY: "${BRAVE_API_KEY}" + max_call_depth: 3 +``` + +| Field | Type | Default | Notes | +|---|---|---|---| +| `name` | string, required | — | Unique; used in logs, metrics, and the `/readyz` body. | +| `url` | string | — | Streamable HTTP endpoint. Set exactly one of `url` / `command`. | +| `command` / `args` | string / []string | — | stdio subprocess launched at gateway startup, kept for the gateway's lifetime. | +| `headers` | map[string]string | `{}` | HTTP transport only. `${VAR}` supported. | +| `env` | map[string]string | `{}` | stdio transport only — the sole credential channel to the subprocess, since it inherits no gateway env. `${VAR}` supported. | +| `allowed_tools` | []string | all | Restricts which discovered tools are exposed to the LLM. | +| `timeout_seconds` | int | 30 | Per-tool-call timeout. | +| `max_call_depth` | int | 5 | Agentic loop depth bound; the minimum positive value across all configured servers applies. | +| `required` | bool | `false` | When `true`, this server's readiness gates `GET /readyz` (`503`, reason "required mcp server unavailable" when unready). Unready means the initialize handshake hasn't completed. Death *after* a successful handshake is detected for **stdio servers only** — an HTTP server that goes unreachable post-handshake keeps reporting ready. | + +Guardrails and the budget plugin run on every turn of the agentic loop, not just the initial request. See [MCP integration](/guides/mcp) for the full tool-execution flow. + +## Compatibility + +Controls how the gateway treats an OpenAI-shaped request parameter that the routed provider can't express (see `GET /v1/capabilities` for the full matrix). + +```yaml +compatibility: + on_unsupported_param: warn # warn | drop | reject ``` -When `mcp_servers` is present, the gateway initialises MCP connections in the background on startup (60-second timeout) and injects available tools into every chat completion request. See [MCP integration](/guides/mcp). +| Value | Behavior | +|---|---| +| `warn` (default) | Forward the parameter anyway and log a warning. | +| `drop` | Remove the parameter from the upstream request and log. | +| `reject` | Fail the request with `HTTP 400` naming the parameter. | + +`warn` and `drop` differ only for providers reached over an OpenAI-compatible request body — there, `warn` genuinely forwards the parameter. A provider with a native wire format (Anthropic, Bedrock, Gemini, Cohere, AI21, Replicate) builds a payload with nowhere to put an unsupported parameter, so `warn` and `drop` send byte-identical upstream requests there and both log "dropping." Use `reject` when a caller needs to know a parameter wasn't honored rather than silently dropped. + +## Observability + +OpenTelemetry tracing. Omit the section entirely (or leave `endpoint` empty with no `OTEL_EXPORTER_OTLP_*` env set) and the gateway runs a zero-allocation NoOp provider at no cost. + +```yaml +observability: + tracing: + enabled: true # tri-state: omit = infer from endpoint, false = hard off, true = force on + endpoint: "" # URL or bare host:port; blank falls back to OTEL_EXPORTER_OTLP_* env + protocol: grpc # grpc | http/protobuf + service_name: ferrogw + sample_ratio: 1.0 # head sampler 0.0-1.0, wrapped in ParentBased + privacy_level: metadata # none | metadata | full + shutdown_grace: 10s # per OTel shutdown stage; total shutdown can take up to 2x this + headers: # OTLP export headers, e.g. vendor API keys + dd-api-key: "${DATADOG_API_KEY}" + + exporters: # plugin observability exporters — none ship in this repo + - name: langsmith + enabled: true + config: + api_key: "${LANGSMITH_API_KEY}" +``` + +The gateway itself reads only two `OTEL_*` environment variables — `OTEL_EXPORTER_OTLP_ENDPOINT` and `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` — and either one takes precedence over `observability.tracing.endpoint` when set. Setting either turns tracing on. `OTEL_TRACES_SAMPLER` has no effect; the sampler is config-only (`sample_ratio`, `ParentBased`) — an inbound request that already carries a sampled `traceparent` is followed regardless of `sample_ratio`. + +`exporters[]` entries reference exporter plugins registered via `observability.RegisterExporter` in a plugin's `init()`; they ship separately in the `ai-gateway-plugins` repo. An unrecognized `name` logs a warning and is skipped — it isn't fatal. + +## `${VAR}` resolution + +Config string values in `plugins[].config`, `mcp_servers[].headers`, `mcp_servers[].env`, `observability.tracing.headers`, and `observability.exporters[].config` support environment variable references. + +- **Only the braced form is a reference.** `${NAME}` (matching `[A-Za-z_][A-Za-z0-9_]*`) is expanded; a bare `$` is literal data — `$100` and `pa$$w0rd` survive byte-for-byte. +- **An undefined variable is a hard error** naming every missing variable — never a silent empty-string substitution. +- **Resolution happens at component construction, not at config load.** The loaded `Config` keeps the literal `${VAR}` text, so a secret never reaches the config-history store, `GET /admin/config`, or a rollback snapshot — and a config pushed through the admin/GitOps API (which never passes through `LoadConfig`) is expanded identically. +- It does **not** apply to core routing fields (`virtual_key`, `mode`, model ids, aliases) — only the free-form/credential-bearing maps listed above. ## Complete example -Copy [`config.example.yaml`](https://github.com/ferro-labs/ai-gateway/blob/main/config.example.yaml) from the repository for a full example covering all 30 providers, aliases, and all 6 built-in plugins. +Copy [`config.example.yaml`](https://github.com/ferro-labs/ai-gateway/blob/main/config.example.yaml) from the repository for a fully annotated example covering all 30 providers, both routing families, all 6 OSS plugins with correct multi-stage registration, MCP (both transports), compatibility, and observability. ## Related pages -- [Routing policies](/guides/routing-policies) — all 8 strategies with examples -- [Plugins](/guides/plugins) — detailed plugin documentation -- [Use cases](/guides/use-cases) — recipe-style configurations -- [MCP integration](/guides/mcp) — tool server setup +- [Routing](/routing) — all 8 strategies with examples +- [Plugins](/plugins) — detailed OSS plugin documentation +- [Enterprise plugins](/plugins/enterprise) — Ferro Labs Managed guardrails (pii-redact, secret-scan, prompt-shield, schema-guard, regex-guard) +- [MCP integration](/guides/mcp) — tool server setup and the agentic loop +- [Provider configuration](/providers/configuration) — per-provider env vars and base URL overrides diff --git a/docs/getting-started/overview.mdx b/docs/getting-started/overview.mdx index 2000f45..0d868e5 100644 --- a/docs/getting-started/overview.mdx +++ b/docs/getting-started/overview.mdx @@ -1,28 +1,53 @@ --- title: Overview -description: Learn what the Ferro Labs AI Gateway does, when to use it, and how it fits into your AI infrastructure stack as a single OpenAI-compatible control plane for all LLM traffic. -keywords: [AI gateway overview, LLM proxy introduction, OpenAI compatible gateway, AI routing layer, AI control plane, multi-provider LLM] +description: "What the Ferro Labs AI Gateway does: a single OpenAI-compatible endpoint for 30 providers, with routing, plugins, an embedded dashboard, and an Admin API." +keywords: [AI gateway overview, LLM proxy introduction, OpenAI compatible gateway, AI routing layer, AI control plane, multi-provider LLM, embedded dashboard] --- -The AI Gateway sits between your applications and upstream model providers. It exposes a single OpenAI compatible API and handles routing, authentication, observability, and resiliency in one place. +The AI Gateway is a single Go binary that sits between your applications and upstream model providers. It exposes one OpenAI-compatible API for chat, streaming, embeddings, and images, and handles routing, authentication, plugins, and observability in one place — including a built-in operations dashboard, so there's nothing else to deploy to see what it's doing. -For a deeper technical view, start with [Architecture](/getting-started/architecture) and [Request lifecycle](/getting-started/request-lifecycle). +For a deeper technical view, continue to [Architecture](/getting-started/architecture) and [Request lifecycle](/getting-started/request-lifecycle). ## When to use it -- You need multiple providers with a consistent interface. -- You want failover and load balancing without rewriting app code. -- You want centralized logs, metrics, and request tracing. +- You call more than one LLM provider and want a single OpenAI-compatible endpoint instead of a client integration per vendor. +- You want failover, load balancing, or cost-aware routing configured in YAML — not rewritten into every service that calls a model. +- You need centralized auth, per-key budgets, and rate limiting in front of every provider, with a scoped Admin API to issue and revoke keys. +- You want one place to see what every request cost, how long it took, and which provider served it — the embedded dashboard, Prometheus metrics, or OpenTelemetry traces, without standing up a separate service. +- You're wiring MCP tool servers into an agentic loop and want the same guardrails and budget checks enforced on every turn, not just the first. + +It matters less if you call exactly one provider directly and have no near-term plan for routing, guardrails, or centralized observability. + +## Core capabilities + +| Capability | What it gives you | Learn more | +|:---|:---|:---| +| Routing | 8 strategies (single, fallback, load balance, least latency, cost-optimized, content-based, A/B test, conditional). `targets` is an allowlist — a provider you've configured only routes traffic if it's listed there. | [Routing](/routing) | +| Providers | 30 providers and 2,500+ models behind one API, with a capability matrix for which OpenAI parameters each one supports. | [Providers](/providers) | +| Plugins & guardrails | 6 built-in plugins — word filter, max-token, response cache, rate limit, budget, request logger — running at `before_request`, `after_request`, and `on_error` stages. | [Plugins](/plugins) | +| Admin API & key management | Scoped API keys (`read_only` / `admin`), dashboard sessions, an audit trail, and config history with rollback. | [Admin API](/api-reference/admin) | +| Embedded dashboard | An operations console compiled into the binary and served from the gateway's own root — no separate image or origin to deploy. | [Dashboard guide](/guides/dashboard) | +| Observability | OpenTelemetry tracing, Prometheus metrics, and request logs carrying `duration_ms`, `ttft_ms`, and `cost_usd`. | [Monitoring](/operations/monitoring) | +| MCP | Connects to MCP tool servers over stdio or Streamable HTTP and drives the agentic `tool_calls` loop itself. | [MCP guide](/guides/mcp) | ## How it works -1. Your app sends OpenAI format requests to the gateway. -2. The gateway applies routing, auth, and policy checks. -3. The gateway forwards traffic to the selected provider. -4. Responses and metrics are recorded and returned to your app. +1. Your app sends an OpenAI-format request to the gateway. `/v1/*` requires a bearer token unless you've explicitly disabled it. +2. Guardrail, auth, and other `before_request` plugins run, then the routing strategy picks a target order. +3. The gateway calls the selected provider, retrying and failing over to the next target per your configuration. +4. `after_request` plugins run, the response is returned to your app, and the request is recorded to metrics, traces, and the request log. + +## Health and readiness + +The gateway exposes three endpoints for orchestrators and monitoring, each answering a different question: + +- **`/livez`** — is the process alive. +- **`/readyz`** — can the gateway actually route traffic; it returns `503` with reason `no routable targets` when none of your configured targets can serve a request. +- **`/health`** — a deeper diagnostic with per-provider status and circuit-breaker state. ## Learn by depth -- **Architecture**: Understand components and system boundaries. -- **Request lifecycle**: Follow each processing stage and fallback path. -- **Guides**: Configure provider auth, routing policies, and controls. +- **[Architecture](/getting-started/architecture)** — components, the embedded dashboard, and system boundaries. +- **[Request lifecycle](/getting-started/request-lifecycle)** — each processing stage and the fallback path. +- **[Providers](/providers)**, **[Routing](/routing)**, **[Plugins](/plugins)** — configure what the gateway routes to and how. +- **[Guides](/guides/dashboard)** — the dashboard, MCP, auth, and other operational tasks. diff --git a/docs/getting-started/quickstart.mdx b/docs/getting-started/quickstart.mdx index 9bdf3bf..9253e30 100644 --- a/docs/getting-started/quickstart.mdx +++ b/docs/getting-started/quickstart.mdx @@ -1,33 +1,86 @@ --- title: Quickstart -description: Get the Ferro Labs AI Gateway running in under 30 seconds with Docker. Send your first OpenAI-compatible request and start routing across multiple LLM providers instantly. -keywords: [AI gateway quickstart, Docker LLM proxy, self-hosted LLM gateway setup, OpenAI proxy install, LLM gateway Docker, AI gateway getting started, Go AI gateway] +description: Install the Ferro Labs AI Gateway, generate a MASTER_KEY with ferrogw init, and send an authenticated OpenAI-compatible chat request to your first provider. +keywords: [AI gateway quickstart, Docker LLM proxy, self-hosted LLM gateway setup, OpenAI proxy install, ferrogw init, LLM gateway Docker, AI gateway getting started, Go AI gateway] --- import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -## Run with Docker +Get the gateway running locally, generate an admin credential, and send an authenticated OpenAI-compatible request in a few minutes. + +## Run the gateway + + + ```bash docker run --rm -p 8080:8080 \ -e OPENAI_API_KEY=sk-your-key \ + -e MASTER_KEY=fgw_your-master-key \ ghcr.io/ferro-labs/ai-gateway:latest ``` -## Build from source +`MASTER_KEY` is the bootstrap admin credential every `/v1/*` and `/admin/*` request authenticates with — pick your own value, or generate one the same way `ferrogw init` does: + +```bash +export MASTER_KEY=fgw_$(openssl rand -hex 16) +``` + +With no `config.yaml` mounted, the gateway builds a default `fallback` config with one target per provider it finds credentials for — enough to route the request below. Mount a real config for retries, aliases, and plugins; see [Configuration](/getting-started/configuration). + + + + +```bash +VER=$(curl -fsSL https://api.github.com/repos/ferro-labs/ai-gateway/releases/latest | grep '"tag_name"' | cut -d'"' -f4) +curl -fsSL "https://github.com/ferro-labs/ai-gateway/releases/download/${VER}/ferrogw_${VER#v}_linux_amd64.tar.gz" | tar xz +chmod +x ferrogw + +export OPENAI_API_KEY=sk-your-key # providers register at startup — export before init/serve +./ferrogw init # scaffolds config.yaml from detected provider keys, prints a MASTER_KEY +export GATEWAY_CONFIG=./config.yaml # a config file is only loaded when this points at it +export MASTER_KEY=fgw_the-key-init-printed +./ferrogw +``` + +`ferrogw init` writes a minimal `config.yaml` naming every provider it found credentials for in the environment (a placeholder target if it found none), and generates the `MASTER_KEY` — printed once, never written to disk. + + + ```bash git clone https://github.com/ferro-labs/ai-gateway.git cd ai-gateway +make build export OPENAI_API_KEY=sk-your-key +./bin/ferrogw init +export GATEWAY_CONFIG=./config.yaml +export MASTER_KEY=fgw_the-key-init-printed make run ``` + + + +## Check it's up + +```bash +curl http://localhost:8080/readyz +``` + +```json +{"status": "ready", "targets": [...]} +``` + +A `503` with `"reason": "no routable targets"` means no provider is both credentialed and named in `targets` — recheck the environment variables above and see [targets is an allowlist](/getting-started/concepts). + +Then open [http://localhost:8080/](http://localhost:8080/) — the operations dashboard is compiled into the binary and served from the same port, no separate container. Sign in with `MASTER_KEY` to browse providers, routing, plugins, and live request logs. + ## Send a request -Replace `model` with a model you have access to. The gateway serves an OpenAI-compatible API at `/v1`, so any OpenAI SDK works by pointing `base_url` at `http://localhost:8080/v1`. +Replace `model` with a model you have access to. Every `/v1/*` request needs a bearer token — `MASTER_KEY`, or an API key created from it — since the gateway requires auth by default. @@ -35,10 +88,11 @@ Replace `model` with a model you have access to. The gateway serves an OpenAI-co ```bash curl http://localhost:8080/v1/chat/completions \ -H "Content-Type: application/json" \ + -H "Authorization: Bearer $MASTER_KEY" \ -d '{ "model": "gpt-4o-mini", "messages": [ - {"role": "user", "content": "Hello from Ferro AI Gateway"} + {"role": "user", "content": "Hello from Ferro Labs AI Gateway"} ] }' ``` @@ -51,11 +105,11 @@ Using the OpenAI SDK (`pip install openai`): ```python from openai import OpenAI -client = OpenAI(base_url="http://localhost:8080/v1", api_key="sk-your-key") +client = OpenAI(base_url="http://localhost:8080/v1", api_key="fgw_your-master-key") response = client.chat.completions.create( model="gpt-4o-mini", - messages=[{"role": "user", "content": "Hello from Ferro AI Gateway"}], + messages=[{"role": "user", "content": "Hello from Ferro Labs AI Gateway"}], ) print(response.choices[0].message.content) ``` @@ -65,11 +119,11 @@ Or the Ferro Labs SDK (`pip install ferrolabsai`): ```python from ferrolabsai import FerroClient -client = FerroClient(base_url="http://localhost:8080/v1", api_key="sk-your-key") +client = FerroClient(base_url="http://localhost:8080/v1", api_key="fgw_your-master-key") response = client.chat.completions.create( model="gpt-4o-mini", - messages=[{"role": "user", "content": "Hello from Ferro AI Gateway"}], + messages=[{"role": "user", "content": "Hello from Ferro Labs AI Gateway"}], ) print(response.content) ``` @@ -84,12 +138,12 @@ import OpenAI from "openai"; const client = new OpenAI({ baseURL: "http://localhost:8080/v1", - apiKey: "sk-your-key", + apiKey: "fgw_your-master-key", }); const response = await client.chat.completions.create({ model: "gpt-4o-mini", - messages: [{ role: "user", content: "Hello from Ferro AI Gateway" }], + messages: [{ role: "user", content: "Hello from Ferro Labs AI Gateway" }], }); console.log(response.choices[0]?.message.content); ``` @@ -101,12 +155,12 @@ import { FerroClient } from "@ferro-labs-ai/sdk"; const client = new FerroClient({ baseUrl: "http://localhost:8080/v1", - apiKey: "sk-your-key", + apiKey: "fgw_your-master-key", }); const response = await client.chat.completions.create({ model: "gpt-4o-mini", - messages: [{ role: "user", content: "Hello from Ferro AI Gateway" }], + messages: [{ role: "user", content: "Hello from Ferro Labs AI Gateway" }], }); console.log(response.choices[0]?.message.content); ``` @@ -114,4 +168,52 @@ console.log(response.choices[0]?.message.content); -If you are using any OpenAI-compatible SDK, set the base URL to `http://localhost:8080/v1` and keep the rest of your code unchanged. +If you are using any OpenAI-compatible SDK, set the base URL to `http://localhost:8080/v1`, the API key to `MASTER_KEY` (or an issued `fgw_...` key), and keep the rest of your code unchanged. + +A successful call returns: + +```json +{ + "id": "chatcmpl-...", + "object": "chat.completion", + "model": "gpt-4o-mini", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "..."}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 12, "completion_tokens": 9, "total_tokens": 21} +} +``` + +## Stream a response + +Set `stream: true` for Server-Sent Events, same as the OpenAI API — every OpenAI SDK's streaming helper works unchanged against the gateway: + +```bash +curl http://localhost:8080/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $MASTER_KEY" \ + -d '{ + "model": "gpt-4o-mini", + "stream": true, + "messages": [{"role": "user", "content": "Count to 5"}] + }' +``` + +```text +data: {"id":"chatcmpl-...","choices":[{"index":0,"delta":{"content":"1"},"finish_reason":null}]} + +data: {"id":"chatcmpl-...","choices":[{"index":0,"delta":{"content":"2"},"finish_reason":null}]} + +... + +data: [DONE] +``` + +## What's next + +- [Configuration](/getting-started/configuration) — write a real `config.yaml`: targets, retry, aliases, plugins +- [Providers](/providers) — register more of the 30 supported providers and see the endpoint matrix +- [Routing](/routing) — fallback, load balance, cost-optimized, and 5 more strategies +- [Plugins](/plugins) — budgets, rate limiting, response caching, and request logging +- [Dashboard guide](/guides/dashboard) — tour the embedded operations console +- [CLI reference](/operations/cli-reference) — `ferrogw validate`, `doctor`, `status`, `admin keys` diff --git a/docs/getting-started/request-lifecycle.mdx b/docs/getting-started/request-lifecycle.mdx index 440d1e1..f964783 100644 --- a/docs/getting-started/request-lifecycle.mdx +++ b/docs/getting-started/request-lifecycle.mdx @@ -1,162 +1,196 @@ --- title: Request lifecycle -description: The real ordered path a request takes through the Ferro Labs AI Gateway — HTTP middleware chain, before/after plugins, routing strategy with circuit breaking, provider call with retry and fallback, the MCP agentic loop, and streaming token metering. -keywords: [AI gateway request lifecycle, LLM proxy request flow, AI middleware pipeline, gateway request processing, plugin pipeline, MCP agentic loop, streaming metering] +description: How a request moves through Ferro Labs AI Gateway — middleware, model admission, plugin stages, the shared routing pipeline, the MCP loop, and errors. +keywords: [AI gateway request lifecycle, LLM proxy request flow, AI middleware pipeline, gateway request processing, plugin pipeline, MCP agentic loop, streaming metering, routing pipeline] --- -This page traces a request through the gateway exactly as the code runs it. The -two anchors are the HTTP middleware chain in -[`internal/httpserver/router.go`](https://github.com/ferro-labs/ai-gateway/blob/main/internal/httpserver/router.go) -and the routing core in -[`gateway.go`](https://github.com/ferro-labs/ai-gateway/blob/main/gateway.go) -(`Route` and `RouteStream`). +This page traces the ordered path a request takes through the gateway: the HTTP +middleware chain, model admission, the plugin stages, the single routing +pipeline shared by chat, streaming, embeddings, and image generation, the +optional MCP tool-calling loop, and how failures are classified into an HTTP +status. ## Overview ```mermaid flowchart TD - C([Client]) --> OTel[OpenTelemetry middleware] - OTel --> Log["Request logging
assigns / propagates X-Request-ID"] - Log --> Rec[Recoverer: panic recovery] - Rec --> RIP["RealIP: rewrite RemoteAddr
from X-Forwarded-For / X-Real-IP"] + C([Client]) --> MW["Recover → OTel → trace ID/X-Request-ID
→ security headers"] + MW --> Probe{"/health /livez /readyz?"} + Probe -- yes --> ProbeResp(["200 / 503
no rate limit, no auth"]) + Probe -- no --> RIP["RealIP: resolve client IP
only from TRUSTED_PROXIES"] RIP --> CORS[CORS] - CORS --> RL{Per-IP rate limit} - RL -- "429 too_many_requests" --> Err1([Error response]) - RL -- allow --> Auth{Auth: ProxyAuth on /v1/*} - Auth -- "401" --> Err2([Error response]) - Auth -- ok --> H[ChatCompletions handler] - - H --> GW[["Gateway.Route /
Gateway.RouteStream"]] - GW --> Alias[Resolve model alias] - Alias --> Before[before_request plugins] - Before -- "plugin sets Skip
e.g. cache hit" --> After - Before --> MCPInj["Inject MCP tools
when servers configured"] - MCPInj --> Strat["Routing strategy selects target
(circuit breaker wraps provider)"] - Strat --> Prov{"Provider call
with retry / fallback"} - Prov -- failure --> OnErr[on_error plugins] - OnErr --> Err3([Error response + failed event]) - Prov -- success --> Loop{"MCP tool_calls?"} - Loop -- yes --> Tools["Execute tools, append results,
re-call provider (bounded depth)"] + CORS --> RL{"Per-IP rate limit
20 rps / burst 40 by default"} + RL -- over --> Err1(["429 rate_limit_exceeded"]) + RL -- ok --> Auth{"Bearer auth
/v1/*, /admin/*, /metrics"} + Auth -- none --> Err2(["401"]) + Auth -- ok --> H[Handler] + + H --> Alias[Resolve model alias] + Alias --> Admit{"admitModel:
does any target serve this model?"} + Admit -- "no (unless a transform plugin is configured)" --> Err3(["404 model_not_found
on_error still runs"]) + Admit -- yes --> Before["before_request plugins
guardrails, transform, rate-limit, budget"] + Before -- reject / error --> Err4(["402 / 429 / 400 / 500
on_error runs"]) + Before -- "SkipProvider (cache hit)" --> After + Before --> MCPInj["Inject MCP tools
only when the caller sent none"] + MCPInj --> Walk[["routeTargets: strategy orders targets,
then retry + circuit breaker + concurrency
per target — every mode, every surface"]] + Walk -- exhausted --> OnErr[on_error plugins] + OnErr --> Err5(["402 / 404 / 429 / 502 / 503 / 504
classified error + failed event"]) + Walk -- success --> Loop{"tool_calls
pending?"} + Loop -- yes --> Tools["Execute tools, append results,
re-run guardrails/budget, re-call provider"] Tools --> Loop Loop -- no --> After[after_request plugins] - After --> Resp(["Response
X-Request-ID trace header + completed event"]) + After --> Resp(["Response
X-Request-ID header + completed event"]) ``` -## 1. HTTP middleware chain - -The chain is assembled in `NewRouter` (`router.go` lines 52–68) and runs in this -order for every request: - -1. **OpenTelemetry** — `gwotel.Middleware` (`router.go:52`). Runs first so any - inbound W3C `traceparent` is extracted into the request context before - logging needs it. With no OTLP provider configured it is a cheap no-op. -2. **Request logging** — `logging.Middleware` (`router.go:53`). Resolves a trace - ID and sets the `X-Request-ID` response header - (`internal/logging/logger.go:94`). Precedence: (1) a trace ID already on the - context from the OTel layer, (2) the inbound `X-Request-ID` header, (3) a - freshly generated 16-byte hex ID. This is what keeps the OTel `trace_id`, the - log trace ID, and the `X-Request-ID` header equal for a request. -3. **Recoverer** — `chimw.Recoverer` (`router.go:54`). Recovers from panics in - downstream handlers and turns them into a 500 instead of crashing the server. -4. **RealIP** — `chimw.RealIP` (`router.go:62`). Rewrites `RemoteAddr` from - `X-Forwarded-For` / `X-Real-IP` so per-IP rate limiting and request logs see - the real client IP rather than the load balancer's. Intended for deployment - behind a trusted proxy. -5. **CORS** — `middleware.CORS(corsOrigins...)` (`router.go:63`). Applies the - configured allowed origins. -6. **Per-IP rate limit** — `middleware.RateLimit(rlStore)` (`router.go:67`), - mounted only when a rate-limit store is configured. A token-bucket keyed on - client IP; over-limit requests get `429` with an OpenAI-shaped - `rate_limit_exceeded` error (`internal/middleware/ratelimit.go`). -7. **Auth** — `middleware.ProxyAuth(store, masterKey)` - (`router.go:195`, applied inside `mountOpenAIRoutes`). Guards the `/v1/*` - routes with bearer-token auth. It delegates to `admin.AuthMiddleware`, unless - `ALLOW_UNAUTHENTICATED_PROXY=true` is set for local dev - (`internal/middleware/proxyauth.go`). - -The first five run for the whole router; rate limit is conditional; auth is -scoped to the OpenAI-compatible `/v1/*` group. `/health`, the dashboard, and -admin routes are mounted separately (`router.go:70–73`). - -## 2. Handler and gateway routing - -`POST /v1/chat/completions` lands in the `ChatCompletions` handler, which calls -`Gateway.Route` (non-streaming) or `Gateway.RouteStream` (when `stream: true`). -`Route` (`gateway.go:397`) executes these stages in order: - -1. **Alias resolution** — `resolveAlias` (`gateway.go:422`) rewrites `req.Model` - to its configured target from `config.Aliases` before any routing decision - (`gateway.go:2085`). -2. **`before_request` plugins** — `runBeforePlugins` (`gateway.go:432`). Runs - guardrails, transforms, and rate-limit-style plugins. If a plugin sets - `Skip` with a response (for example a cache hit), `after_request` plugins - still fire and that early response is returned immediately - (`gateway.go:382`). A plugin error increments the `Rejected` metric and - aborts. -3. **Routing strategy selects a target** — `getStrategy` (`gateway.go:427`, - built in `gateway.go:890`) constructs the strategy from - `config.strategy.mode`: `single`, `fallback`, `loadbalance`, `latency`, - `cost-optimized`, `conditional`, `content-based`, or `ab-test`. The provider - lookup closure transparently wraps each target in a **circuit breaker** - (`cbProvider`, `gateway.go:1030`): an open circuit short-circuits with - `ErrCircuitOpen` instead of calling the provider. -4. **Provider call with retry / fallback** — `s.Execute` (`gateway.go:489`) - performs provider selection and the actual upstream call. In `fallback` mode, - per-target retry (attempts, status codes, backoff) is applied - (`gateway.go:938`) and the strategy advances to the next target on eligible - failures. -5. **`after_request` plugins** or **`on_error` plugins** — on success, - `after_request` plugins run (logging, caching) and may rewrite the response - (`gateway.go:579`). On failure, `on_error` plugins run, the error is - classified (`circuit_open` vs `provider_error`), metrics and a `failed` - lifecycle event are emitted, and the error is returned (`gateway.go:495`). -6. **Response** — `recordSuccess` (`gateway.go:595`) emits Prometheus metrics, - computes cost from the model catalog, stamps the trace span, and dispatches - the `completed` event. The response carries the `X-Request-ID` trace header - set back in the logging middleware. - -## 3. MCP agentic loop - -When MCP servers are configured, the gateway injects their tool definitions into -the request before routing (`gateway.go:450`), de-duplicating against any tools -the caller already supplied. After the first provider response, the agentic loop -runs (`gateway.go:544`): - -- `ShouldContinueLoop` checks whether the model returned `tool_calls` and the - call depth is under the configured limit. -- `ResolvePendingToolCalls` executes the requested tools and returns the - assistant message plus one tool-result message per call. -- Those messages are appended to `req.Messages` and the provider is called again - (`gateway.go:565`). - -The loop repeats until the model stops emitting `tool_calls` or the depth limit -is reached. Intermediate calls are forced non-streaming so each response can be -inspected for tool calls (`gateway.go:477`). - -## 4. Streaming path - -`RouteStream` (`gateway.go:1163`) handles `stream: true`: - -- **MCP redirect** — if MCP servers are registered, the request is routed - through `Route` so the full agentic loop runs to completion, then the final - response is wrapped into a **single final chunk** via `responseStream` - (`gateway.go:1200`, `gateway.go:1420`). This is the current Phase 1 behavior: - the streaming caller still sees `stream: true` in emitted events, but receives - one terminal `chat.completion.chunk`. -- **Normal streaming** — `before_request` plugins run, then - `resolveStreamingProviderLocked` (`gateway.go:1242`) orders targets per - strategy mode and picks a circuit-breaker-aware streaming provider. - `sp.CompleteStream` opens the upstream channel (`gateway.go:1278`). -- **Token metering** — the raw channel is wrapped by `streamwrap.Meter` - (`gateway.go:1417`), which counts input/output tokens, computes cost from the - catalog, records TTFT/TTLT timings, runs `after_request` plugins through its - `CompletionFn`, updates circuit-breaker state, and emits Prometheus metrics - plus the `completed` or `failed` event once the stream drains. +## HTTP middleware chain + +Every request runs through the root chain in order: panic recovery that still +returns the gateway's JSON error envelope, an OpenTelemetry layer that extracts +an inbound W3C `traceparent` (a no-op with no OTLP endpoint configured), a +logging layer that assigns or propagates the trace ID and sets the +`X-Request-ID` response header, and baseline security headers (CSP, +`X-Frame-Options`, HSTS on TLS). + +`/health`, `/livez`, and `/readyz` are mounted at this point — deliberately +**ahead of** IP resolution, rate limiting, and auth, so a traffic burst against +`/v1/*` that exhausts one IP's bucket can never also 429 an orchestrator's +liveness probe. Every other route sits behind an additional chain: + +1. **Client IP resolution** — `X-Forwarded-For` / `X-Real-IP` are honored only + when the direct TCP peer falls inside a `TRUSTED_PROXIES` CIDR (default: + loopback only). When trusted, the forwarded chain is read **from the right** + — the hop nearest the gateway — so a caller cannot spoof their address by + prepending fake entries to the header. +2. **CORS**, applying the configured allowed origins. +3. **Per-IP rate limit** — **on by default** (20 requests/sec, burst 40, keyed + on the resolved client IP, up to 100,000 tracked IPs). `RATE_LIMIT_RPS=0` + removes this middleware entirely. +4. **Auth** — a bearer token is required on `/v1/*` (unless + `ALLOW_UNAUTHENTICATED_PROXY=true`), on every `/admin/*` route, and on + `/metrics` (`read_only` or `admin` scope; `/debug/*` needs `admin`). + +## Model admission and before_request plugins + +`POST /v1/chat/completions` resolves any configured model alias, then the +gateway asks **admitModel**: does any configured target serve this model at +all? This runs *before* the plugin stage on purpose — a rate limiter spends a +token and a budget spends money, so a model no target can ever reach must not +be allowed to spend either on its way to a 404. A refusal here still runs +`on_error`, so the denial is recorded like any other. The check is skipped when +a `before_request` plugin reports itself a transform, since a transform is +exactly what can turn an unroutable alias into a routable model id. + +`before_request` plugins then run in configured order (guardrails, transforms, +rate-limit, budget). A `Reject` verdict ends the stage with a 4xx/429/402; a +plugin **error** fails closed (500) for every type except logging and metrics, +which fail open. + +:::note SkipProvider, not Skip +`Context.Skip` was removed. The response-cache plugin (and anything similar) +now sets **`Context.SkipProvider`**, which suppresses only the upstream +provider call. Every remaining `before_request` plugin still runs, and so does +the whole `after_request` stage — a cache hit can no longer disable a guardrail, +rate limiter, or budget check behind it. +::: + +When MCP servers are configured, their tool definitions are added to the +request — but **only when the caller supplied no `tools` of their own**. A +request that already carries tools passes through untouched; the gateway never +merges or de-duplicates against it. + +## Routing pipeline: one walk for every surface + +Chat, streaming, embeddings, and image generation all route through the same +walk. A routing strategy (`single`, `fallback`, `loadbalance`, +`least-latency`, `cost-optimized`, `conditional`, `content-based`, `ab-test`) +decides target **order** only — retry, circuit breaking, concurrency limiting, +and error classification live once in the pipeline, so the four surfaces +cannot drift from each other. + +For each candidate target the pipeline applies, in this order: the target's +`retry` policy, its circuit breaker, then its concurrency limiter, then the +provider call. + +**`targets[].retry` is honored under every routing mode** — not only +`fallback`. It controls how many times the pipeline re-asks the *same* target +(default: 1 attempt, i.e. no retry, if `attempts` is unset or `0`). Whether the +walk tries a **different** target after that is a separate question, decided +by the mode: + +| Mode family | Modes | On a failed target | +|---|---|---| +| Pool | `fallback`, `loadbalance`, `least-latency`, `cost-optimized`, `ab-test` | Advances to the next candidate | +| Named | `single`, `conditional`, `content-based` | Commits to the one target and reports the failure | + +Both families skip a target whose circuit breaker is **open** before +committing to it — including named modes, where the operator picked that +target specifically. When every eligible candidate's circuit is open, the walk +still attempts one anyway rather than reporting a false 404, and the breaker +turns that attempt into a `503`. + +## MCP agentic loop + +When MCP participates and the provider's response carries `tool_calls`, the +gateway executes the requested tools, appends the assistant message and the +tool results to the conversation, and re-calls the provider — repeating until +no `tool_calls` remain or the configured depth limit is reached. Each +intermediate call is forced non-streaming so the response can be inspected. + +Every loop turn is a real provider call, so every turn faces the plugins that +bound one: `before_request` guardrails, the rate-limit plugin, and the budget +plugin all re-run per turn (transform, logging, and metrics plugins are +skipped, so the model isn't rewritten mid-conversation and nothing is +double-counted). Budget's per-turn check includes what this request has +already spent, so a long tool-calling conversation can be cut off by the spend +cap mid-loop. Token usage and cost accumulate across every turn and are +reported on the final response. + +## Streaming path + +When MCP servers are registered, the caller sent no `tools` of its own, and the +registry has discovered tools, a `stream: true` request is redirected through +the same non-streaming path described above so the full agentic loop can run +to completion — the final response is then wrapped into a single terminal +`chat.completion.chunk`. A request that supplies its own `tools` bypasses MCP +entirely and streams normally, unaffected by any configured server. + +Otherwise: model admission and `before_request` plugins run exactly as above, +then the pipeline selects and starts a streaming-capable target under the same +retry, breaker, and concurrency rules. This is the only safe retry window — +once the provider's stream channel is handed back, nothing is retried or +replayed. The raw channel is then wrapped by a metering layer that counts +input/output tokens, computes cost from the model catalog, records +time-to-first-token and total latency, runs the `after_request` stage once the +stream completes, updates the circuit breaker's outcome, and emits Prometheus +metrics plus the `completed` or `failed` event when the stream drains. + +## Error classification and on_error + +`on_error` plugins always run on a failure — a model-admission refusal, a +`before_request` rejection or plugin error, an `after_request` plugin error, or +an exhausted routing walk — so a denied or failed request is never missing from +the log. + +| Status | Code | Cause | +|---|---|---| +| 402 | `insufficient_quota` | The budget plugin's spend cap is exhausted | +| 404 | `model_not_found` | No configured target serves the model | +| 429 | `rate_limit_exceeded` | A rate-limit-type plugin denial, or the upstream itself returned 429 | +| 429 | `provider_saturated` | The target's concurrency limit and queue are both full | +| 503 | `upstream_unavailable` | The target's circuit breaker is open | +| 504 | `gateway_timeout` | The gateway's `request_timeout` (or the caller's own deadline) elapsed | +| 502 | `upstream_auth_error` / `upstream_error` | Upstream returned 401/403, or an unclassified 5xx | +| 400 | `unsupported_parameter` | `compatibility.on_unsupported_param: reject` matched an unsupported field | + +An upstream `400`/`422`/`404` passes through with the provider's own message — +the one case where upstream text reaches the caller — because it describes the +request's own shape and is the caller's to fix. ## Related pages - [Architecture](/getting-started/architecture) - [Configuration](/getting-started/configuration) -- [Routing policies](/guides/routing-policies) -- [Plugins](/guides/plugins) +- [Routing](/routing) +- [Plugins](/plugins) +- [API errors](/api-reference/errors) - [Operations monitoring](/operations/monitoring) diff --git a/docs/guides/admin-auth.mdx b/docs/guides/admin-auth.mdx deleted file mode 100644 index a6b980c..0000000 --- a/docs/guides/admin-auth.mdx +++ /dev/null @@ -1,43 +0,0 @@ ---- -title: Admin authentication -description: Secure the Ferro Labs AI Gateway admin API with scoped bearer token authentication — create, rotate, and revoke admin keys for managing gateway configuration and logs. -keywords: [AI gateway admin auth, LLM proxy admin API, bearer token gateway, admin API security, gateway admin keys] ---- - -Admin endpoints require a bearer token in the `Authorization` header. - -```bash -Authorization: Bearer -``` - -## Scopes - -- `admin` - full access -- `read_only` - read-only access to admin endpoints - -## Primary credential: MASTER_KEY - -The primary admin credential is the **`MASTER_KEY`**. Run `ferrogw init` to -generate one, then export it before starting the gateway: - -```bash -export MASTER_KEY="ferro-..." # generated by `ferrogw init` -``` - -Use the `MASTER_KEY` as the bearer token to access admin endpoints (full admin -scope). From there you can issue scoped, persistent API keys via -`POST /admin/keys` for `admin` or `read_only` access. - -:::note Deprecated: bootstrap keys -The legacy `ADMIN_BOOTSTRAP_KEY` / `ADMIN_BOOTSTRAP_READ_ONLY_KEY` env vars are -**deprecated** and emit deprecation warnings at startup. They are honored only on -first run while the API key store is empty **and** `MASTER_KEY` is unset: - -```bash -export ADMIN_BOOTSTRAP_KEY=change-me -export ADMIN_BOOTSTRAP_READ_ONLY_KEY=change-me -export ADMIN_BOOTSTRAP_ENABLED=true -``` - -Use `MASTER_KEY` instead. -::: diff --git a/docs/guides/auth.mdx b/docs/guides/auth.mdx index f0e36ad..10006b8 100644 --- a/docs/guides/auth.mdx +++ b/docs/guides/auth.mdx @@ -1,46 +1,86 @@ --- title: Authentication -description: Configure provider credentials, gateway access keys, and authentication strategies in the Ferro Labs AI Gateway to secure your LLM proxy and control access to AI providers. -keywords: [AI gateway authentication, LLM proxy auth, API key gateway, provider credentials, gateway access control, bearer token auth] +description: How Ferro Labs AI Gateway authenticates requests — MASTER_KEY bootstrap, fgw_-prefixed API keys, dashboard sessions, scopes, and production-mode safety checks. +keywords: [AI gateway authentication, LLM proxy auth, API key gateway, bearer token auth, dashboard session, admin scopes] --- -## Provider credentials +The gateway has no user accounts — a bearer token is the credential, and every +authenticated route (admin API, `/v1/*` data plane, `/metrics`, `/debug/*`) +checks the same credential chain: `MASTER_KEY`, a stored API key, or a +dashboard session minted from either. -Set provider specific credentials as environment variables. The gateway injects these when proxying requests to providers. +## Provider credentials -Example for OpenAI: +Set provider-specific credentials as environment variables. The gateway +injects these when proxying requests upstream; a provider is only registered +when its required variable is present. ```bash export OPENAI_API_KEY=sk-your-key ``` -## Gateway API keys +## MASTER_KEY: bootstrap and break-glass -The gateway's primary credential is the **`MASTER_KEY`**. It is a single admin -credential used for both the admin API and client (proxy) requests. Run -`ferrogw init` to generate one, then export it: +`MASTER_KEY` is the credential that gets you started and the way back in if +every stored key is lost — not a daily login. Generate one and export it +before starting the gateway: ```bash -export MASTER_KEY="ferro-..." # generated by `ferrogw init` +ferrogw init +export MASTER_KEY="fgw_..." # generated by `ferrogw init` ``` -Once a `MASTER_KEY` is set you can use it as a bearer token to call the admin API -and issue scoped, persistent gateway API keys via `POST /admin/keys`. Those issued -keys can then be used by clients in place of the master key. +`MASTER_KEY` is compared with a constant-time check and authenticates as a +synthetic admin-scoped key (`master-key:`). It has **no row in +the key store**, so unlike a stored key it cannot be revoked or expired +without restarting the process — rotating or unsetting the value invalidates +it and any session minted from it immediately, since the check re-derives the +fingerprint on every request. -:::note Deprecated: bootstrap keys -The legacy `ADMIN_BOOTSTRAP_KEY` / `ADMIN_BOOTSTRAP_READ_ONLY_KEY` env vars are -**deprecated** and emit deprecation warnings at startup. They are honored only on -first run while the API key store is empty **and** `MASTER_KEY` is unset. Use -`MASTER_KEY` instead. +:::tip Give each operator their own key +Use `MASTER_KEY` to bootstrap, then create one admin-scoped key per operator +via `POST /admin/keys` or the dashboard. A shared key has to be rotated and +redistributed to everyone when one person leaves; a per-operator key is +revoked on its own and its actions are attributable in the audit trail. See +[Virtual keys and API keys](/guides/virtual-keys) for key management. ::: +## Issuing scoped API keys + +Once `MASTER_KEY` (or an admin-scoped key) is set, use it as a bearer token to +call the admin API and issue persistent, scoped keys: + +```bash +curl -X POST http://localhost:8080/admin/keys \ + -H "Authorization: Bearer $MASTER_KEY" \ + -H "Content-Type: application/json" \ + -d '{"name": "ci-pipeline", "scopes": ["admin"]}' +``` + +Issued keys are prefixed **`fgw_`** (64 hex characters) — never `ferro-...`. +The full secret is returned only once, in the creation response; the store +keeps a SHA-256 hash. + +Two scopes exist: `admin` (full access) and `read_only`. **Omitting `scopes` +on creation defaults to `read_only`** — least privilege by default, not +`admin`. A caller wanting an admin-scoped key must request it explicitly: + +```bash +# Defaults to read_only — scopes omitted +curl -X POST http://localhost:8080/admin/keys \ + -H "Authorization: Bearer $MASTER_KEY" \ + -H "Content-Type: application/json" \ + -d '{"name": "monitoring-scraper"}' +``` + +Naming a scope outside `{admin, read_only}` fails with `400 invalid_scope`. + ## Client request authentication -By default, **all `/v1/*` inference routes require authentication** +By default, **all `/v1/*` routes require authentication** (`/v1/chat/completions`, `/v1/completions`, `/v1/embeddings`, -`/v1/images/generations`, `/v1/models`, and the pass-through proxy). Clients must -send a bearer token — either the `MASTER_KEY` or an admin-issued API key: +`/v1/images/generations`, `/v1/models`, and the pass-through proxy). Clients +send a bearer token — either `MASTER_KEY` or an issued `fgw_` key: ```bash curl http://localhost:8080/v1/chat/completions \ @@ -49,6 +89,83 @@ curl http://localhost:8080/v1/chat/completions \ -d '{"model":"gpt-4o","messages":[{"role":"user","content":"Hello"}]}' ``` +Data-plane requests accept the same credential chain as the admin API — +including a dashboard session token — so the embedded dashboard's Playground +authenticates identically to a CLI client. + To disable proxy authentication for **local development only**, set `ALLOW_UNAUTHENTICATED_PROXY=true`. The gateway logs a warning on startup when -this is enabled; it is not recommended for production. +this is enabled, and [production mode](#production-mode) refuses to start with +it set. + +## Dashboard sessions + +The embedded dashboard doesn't hold a raw API key in the browser. It exchanges +one for a short-lived session token: + +```bash +curl -X POST http://localhost:8080/admin/session \ + -H "Authorization: Bearer $MASTER_KEY" +``` + +`POST /admin/session` is deliberately unauthenticated as a *route* — the +bearer token presented in the request body's place is the credential being +validated, so this is how a caller obtains the session token every other +`/admin/*` call needs. The minted session: + +- carries only the scopes of the credential that minted it +- expires after **24 hours absolute or 1 hour idle**, whichever comes first +- is revocable per-session (`DELETE /admin/sessions/{id}`) or all at once + (`DELETE /admin/sessions`) +- is stored hashed, following the same backend as the API key store + (`API_KEY_STORE_BACKEND`) + +Sign-in attempts against `POST /admin/session` are throttled independently of +the general per-IP rate limit, so tuning `RATE_LIMIT_RPS` for inference +traffic can't loosen this endpoint. Every attempt — not only failures — +consumes a token, since a flood of valid requests costs the same store lookup +and hash comparison as an invalid one. Accepted and denied sign-ins are +recorded in the audit trail. + +## Scope requirements by route + +| Route | Required scope | +|---|---| +| `/v1/*` (data plane) | any valid credential, unless `ALLOW_UNAUTHENTICATED_PROXY=true` | +| `/admin/*` (except `POST /admin/session`) | `read_only` or `admin`, depending on the operation | +| `/metrics` | `read_only` or `admin` | +| `/debug/*` (`/debug/vars`, `/debug/pprof/*`) | `admin` only | +| `/health`, `/livez`, `/readyz` | none (unauthenticated) | + +`/metrics` and `/debug` are split into two tiers on purpose: a monitoring +system scraping `/metrics` and an engineer pulling a heap or goroutine profile +are different actors, and a profile can hold request bodies, prompts, and +credentials — the reason `/debug/*` is admin-only rather than sharing the +read-only tier `/metrics` uses. + +## Production mode + +Setting `GATEWAY_ENV=production` turns on startup safety checks that split +into two tiers: + +**Refused — the gateway exits rather than start:** + +- `ALLOW_UNAUTHENTICATED_PROXY=true` — every `/v1/*` route would be + unauthenticated +- `CORS_ORIGINS` containing `*` — matched literally against `Origin`, so it + would allow no cross-origin request while reading as though it allowed all + of them + +**Warned — logged, startup continues:** + +- `RATE_LIMIT_RPS=0` +- `ENABLE_PPROF=true` +- the in-memory API key store (operator keys, dashboard sessions, and the + audit trail are lost on restart) + +## Related + +- [Virtual keys and API keys](/guides/virtual-keys) — key lifecycle, expiry, and revocation +- [Rate limiting](/guides/rate-limiting) — per-IP and per-key limits +- [Server settings](/operations/server-settings) — `GATEWAY_ENV`, `TRUSTED_PROXIES`, and other env vars +- [Admin API reference](/api-reference/admin) — full `/admin/*` endpoint contract diff --git a/docs/guides/cost-tracking.mdx b/docs/guides/cost-tracking.mdx index 3db2cde..f49950e 100644 --- a/docs/guides/cost-tracking.mdx +++ b/docs/guides/cost-tracking.mdx @@ -1,7 +1,7 @@ --- title: Cost Tracking -description: See and cap LLM spend in the Ferro Labs AI Gateway using model-catalog pricing, the ferro.cost.usd trace attribute, per-key budget counters, and spend limits. -keywords: [cost tracking, llm spend, model catalog pricing, ferro.cost.usd, budget plugin, spend limit] +description: Track LLM spend in the Ferro Labs AI Gateway with model-catalog pricing, ferro.cost.* trace attributes, durable per-request cost_usd, and the 402 budget plugin. +keywords: [cost tracking, llm spend, model catalog pricing, ferro.cost.usd, budget plugin, spend limit, insufficient_quota] --- Cost tracking in the Ferro Labs AI Gateway has two halves: **seeing** what each @@ -11,11 +11,11 @@ ingredient — per-model token prices from the model catalog. ## Where prices come from Prices live in the **model catalog** (`models/catalog.go`). Each model carries a -`Pricing` block whose two core fields are token rates in USD **per one million -tokens**: +`Pricing` block whose fields are token rates in USD **per one million tokens**: - `input_per_m_tokens` — cost per 1M prompt tokens - `output_per_m_tokens` — cost per 1M completion tokens +- `cache_read_per_m_tokens` / `cache_write_per_m_tokens` — prompt-cache rates These are pointers: a `nil` rate means the field does not apply to that model's mode — it does **not** mean free. Use `0` for genuinely free models. @@ -23,10 +23,14 @@ mode — it does **not** mean free. Use `0` for genuinely free models. The catalog loads from a remote release with an embedded fallback: 1. **Remote** — fetched from the latest `model-catalog` GitHub release - (`catalog.json`) with a 1-second timeout. + (`catalog.json`) during startup, before the listener binds, with a + **10-second** timeout by default. Override it with + `FERRO_MODEL_CATALOG_TIMEOUT` (a Go duration); set it to `0` to skip the + remote fetch entirely — useful for air-gapped deployments that only want + the embedded catalog. 2. **Embedded fallback** — a bundled `catalog_backup.json` compiled into the - binary, used whenever the remote fetch or parse fails. The gateway never - fails to start because the catalog is unavailable. + binary, used whenever the remote fetch or parse fails, or is skipped. The + gateway never fails to start because the catalog is unavailable. 3. **24h refresh** — a background ticker reloads the catalog every 24 hours; a failed refresh keeps the currently loaded catalog. @@ -35,8 +39,22 @@ useful for air-gapped deployments or enterprise custom pricing: ```bash export FERRO_MODEL_CATALOG_URL="https://pricing.internal/catalog.json" +export FERRO_MODEL_CATALOG_TIMEOUT="0" # skip the remote fetch entirely ``` +### Prompt-cache pricing + +A provider reports a cached prompt as `PromptTokens` **inclusive** of +`CacheReadTokens`. When a catalog entry sets `cache_read_per_m_tokens`, the +cached subset is billed at that rate and the remainder at the input rate, so +it is never billed twice. When a catalog entry sets no cache-read rate, the +whole prompt bills at the input rate — an unpriced dimension bills as a +visible over-report rather than silently as zero. `cache_write_per_m_tokens` +prices `CacheWriteTokens`, which sit outside `PromptTokens` and cost nothing +unless the rate is set. This is the same rule `models.Calculate` applies for +every catalog-priced cost figure below, and the same rule the `budget` plugin +applies to its own operator-configured rates (see [Capping spend](#capping-spend)). + ## Seeing spend ### Per-request cost on traces @@ -58,11 +76,47 @@ synchronously and stamps it onto the completed-request span and event as the Export these to any OTLP backend — see [Observability](/guides/observability) — to chart spend per model, per route, or per tenant. +### Durable spend: `cost_usd` on request-log rows + +With the `request-logger` plugin's `persist: true` and a configured request-log +store (`REQUEST_LOG_STORE_BACKEND`/`_DSN`, sqlite or postgres), every logged +row carries a `cost_usd` column — the same catalog-priced estimate as +`ferro.cost.usd`, computed once and reused. `cost_usd` is `null` when the +catalog does not price the routed model, which is a coverage gap, not a claim +the request was free. + +```bash +curl "http://localhost:8080/admin/logs?limit=20" \ + -H "Authorization: Bearer $MASTER_KEY" +``` + +### `GET /admin/logs/stats` — spend by provider and model + +The stats endpoint aggregates the persisted log into per-dimension spend, so +"what did the last week cost, broken down by provider and model" is one +authenticated call rather than a trace-store query: + +```bash +curl "http://localhost:8080/admin/logs/stats?since=2026-08-01T00:00:00Z" \ + -H "Authorization: Bearer $MASTER_KEY" +``` + +The response includes a `summary.cost_usd` total (with `unpriced_requests` +counting completed requests the catalog could not price — a floor on real +spend, not a total), plus `by_provider` and `by_model` maps, each entry +carrying its own `cost_usd`, request count, error count, and token totals. +This is the same data source the dashboard's **Analytics** page charts — see +[Dashboard](/guides/dashboard) for the UI. Both the log store and the stats +rollup require the request-log store to be configured; without it, spend +visibility falls back to traces (above) and the budget plugin's in-memory +counters (below). + ### Per-key counters from the budget plugin When the `budget` plugin is enabled, it accumulates USD spend per API key in an in-memory store. Those counters drive enforcement (below) and reflect live spend -since process start. They are in-memory only and do not survive a restart. +since process start. They are **in-memory only** and do not survive a restart — +for spend that does, use `cost_usd` on request-log rows. ### `GET /admin/keys/usage` @@ -70,15 +124,16 @@ The admin usage endpoint returns per-key activity, sorted and filterable, with a rolling summary: ```bash -curl http://localhost:8080/admin/keys/usage?sort=usage \ +curl "http://localhost:8080/admin/keys/usage?sort=usage" \ -H "Authorization: Bearer $MASTER_KEY" ``` It supports `sort` (`usage` or `last_used`), `active`, `since`, `limit`, and `offset`, and returns a `summary` with `total_keys`, `active_keys`, `total_usage`, and `returned_keys`. Note that this endpoint reports per-key -**request counts** and last-used timestamps — for USD spend, read the -`ferro.cost.usd` traces or the budget plugin counters. +**request counts** and last-used timestamps, not USD — for spend per key, read +the `ferro.cost.usd` traces, `GET /admin/logs?api_key_id=` for that key's +priced rows, or the budget plugin's counters. ## Capping spend @@ -101,6 +156,10 @@ plugins: # Pricing used to calculate cost from token counts in the response. input_per_m_tokens: 3.0 # USD per 1 million prompt tokens output_per_m_tokens: 15.0 # USD per 1 million completion tokens + # Optional: unset bills the cached subset at the input rate (visible + # over-report, never silently free); set to price it separately. + cache_read_per_m_tokens: 0.30 + cache_write_per_m_tokens: 3.75 # Maximum number of API keys tracked in memory. Evicts lowest-spend key at cap. max_keys: 10000 @@ -113,27 +172,64 @@ plugins: spend_limit_usd: 10.0 input_per_m_tokens: 3.0 output_per_m_tokens: 15.0 + cache_read_per_m_tokens: 0.30 + cache_write_per_m_tokens: 3.75 + max_keys: 10000 ``` When a key's accumulated spend reaches `spend_limit_usd`, the `before_request` -check rejects the request with HTTP 429. The plugin sets its own -`input_per_m_tokens` / `output_per_m_tokens` rather than reading the catalog, so -it stays self-contained; pick rates that match the models you route. If -`spend_limit_usd` is set but both rates are `0`, the plugin refuses to start — -cost would always be `0` and the limit would never bite. +check rejects the request with **HTTP 402 `insufficient_quota`** — not 429. +Waiting does not restore a spend cap the way it restores a rate-limit token, so +a 429's retry hint would just send every SDK into a backoff schedule it was +always going to exhaust; 402 tells clients to stop retrying instead. + +The plugin sets its own `input_per_m_tokens` / `output_per_m_tokens` (and +optional cache rates) rather than reading the catalog, so it stays +self-contained; pick rates that match the models you route. If +`spend_limit_usd` is set but every rate (input, output, and both cache rates) +is `0`, the plugin refuses to start — cost would always be `0` and the limit +would never bite. + +Two things worth knowing about how the check runs: + +- **Soft cap, not a reservation.** The `before_request` check only reads + already-committed spend; it never reserves the request's eventual cost. A + bounded number of concurrently in-flight requests for the same key can all + pass the check and collectively overshoot the limit once each completes — + bounded by in-flight count × per-request cost, not unbounded. A hard + pre-authorization cap is deliberately out of scope: a reservation that leaks + on every error, cancellation, or circuit-open response would permanently pin + a key at its cap. +- **Per-turn, inside an agentic tool loop.** Guardrail and budget plugins + re-run at `before_request` on every MCP tool-loop turn. The budget check adds + the loop's running cost so far (`Measurements.CostUSD`) to the stored spend + before comparing against the limit, so a key can be cut off **mid-loop** + rather than only between requests — the loop is the one place a single + request can spend without bound. + +Cache-served responses (from the `response-cache` plugin) skip cost recording +entirely: nothing was billed upstream, so nothing is added to the key's spend. :::warning OSS enforcement caveat Per-key budget enforcement requires the API key to be present at -`pctx.Metadata["api_key"]`. In bare OSS that field is **not** populated, so the -plugin is inert and tracks nothing. It becomes active when a host populates the -metadata — as in Ferro Labs Managed. For durable, billing-grade enforcement that -survives restarts, use the managed server-side budget controls. +`pctx.Metadata["api_key"]`. In bare OSS that field is populated whenever a +request carries a bearer token — issued key or `MASTER_KEY` — validated by the +gateway's own auth middleware, so budget tracking is live for any authenticated +deployment (the default; see [Auth](/guides/auth)). Requests made under +`ALLOW_UNAUTHENTICATED_PROXY=true` carry no key and are not tracked or +rejected by this plugin. The counters themselves are in-memory and reset on +restart — for spend enforcement that survives a restart, use the durable +`cost_usd` request-log data above alongside your own alerting, or Ferro Labs +Managed's server-side budget controls. ::: ## Related +- [Dashboard](/guides/dashboard) — the embedded Analytics page charting spend, + tokens, and latency from the same `/admin/logs/stats` data. - [Observability](/guides/observability) — exporting `ferro.cost.usd` and other attributes to OTLP backends. -- [Plugins](/guides/plugins) — plugin stages and the built-in plugin set. +- [Budget plugin](/plugins/budget) — full config reference and validation rules. +- [Plugins](/plugins) — plugin stages and the built-in plugin set. - [Rate limiting](/guides/rate-limiting) — capping request volume rather than dollar spend. diff --git a/docs/guides/dashboard.mdx b/docs/guides/dashboard.mdx new file mode 100644 index 0000000..de36af6 --- /dev/null +++ b/docs/guides/dashboard.mdx @@ -0,0 +1,72 @@ +--- +title: The embedded dashboard +description: "Tour the embedded Ferro Labs AI Gateway dashboard: a React console served at the gateway root with overview, analytics, request logs, config and playground." +keywords: [ferro labs dashboard, ai gateway dashboard, operations console, embedded spa, admin session, request logs, analytics, playground] +--- + +Every Ferro Labs AI Gateway binary serves a built-in operations console. As of **v1.4.0** it is a React single-page app compiled into the binary with `go:embed` (`web/` builds the bundle, `internal/webui` serves it) and mounted at the **site root on the gateway's own port** — the same origin as the API. There is no second container, no separate web image, no `/dashboard` path, and no `GATEWAY_BASE_URL`: the SPA answers every path the API router does not, so an unmatched route lands on the console. + +Sign in and it reads the live gateway — traffic, spend, providers, routing, plugins, request logs, and the audit trail — over the same `/admin/*` endpoints the CLI uses. + +## Accessing the dashboard + +Start the gateway and open its root in a browser: + +```bash +ferrogw # starts the server on :8080 by default +# then open +open http://localhost:8080/ +``` + +For a self-hosted deployment, that is `http://your-gateway/` on whatever host and port you run it on — the console lives wherever the API does. + +### Sign in with a key, browse with a session + +The gateway has no user accounts; a key *is* the identity. The login screen takes your `MASTER_KEY` or any admin / `read_only` API key and exchanges it for a short-lived session via `POST /admin/session`. The credential is sent exactly once and **never stored in the browser** — only the returned session token reaches storage. That is the whole point of the exchange: a master key sitting in browser storage cannot be revoked or expired, but a session can. + +Sessions are **24h absolute / 1h idle**. They are listable and revocable from the **API Keys** page (backed by `GET /admin/sessions`, `DELETE /admin/sessions/{id}`), so a single lost device can be signed out without disturbing every operator, and `DELETE /admin/sessions` signs everyone out at once. `POST /admin/session` is throttled per source address to blunt credential guessing. + +:::tip +Give each operator their own admin-scoped key from the **API Keys** page and keep `MASTER_KEY` for bootstrap and break-glass. Sign-ins and key changes are recorded in the audit trail against the acting credential — a shared key names itself and answers nothing. +::: + +## The pages + +The left navigation maps one-to-one onto the gateway's admin surface: + +| Page | What it shows | +|------|---------------| +| **Overview** | At-a-glance health: routable targets, recent traffic, and spend, read from the live gateway. | +| **Analytics** | Traffic and failure series, token split (prompt vs completion), **p50 / p95 / p99** request latency and **TTFT**, and spend ranked per provider and per model — all from `GET /admin/logs/stats`. | +| **Providers** | The registered providers, their configured surfaces, and per-target **circuit state** (polled from `/health`). | +| **Routing Strategy** | The active routing mode and its targets. | +| **Plugins** | The plugins this instance has configured, described against the catalog the build ships (`GET /admin/plugins/catalog`) so a card can never name a setting that does not exist. | +| **Request Logs** | One row per request, filterable by model, provider, and credential. It resolves each row's `api_key_id` back to the key **name** (via `GET /admin/keys`), marks a key revoked or expired since it served, and shows `duration_ms`, `ttft_ms`, and `cost_usd`. The credential itself is never displayed. | +| **Audit** | The durable audit trail from `GET /admin/audit` — credential changes, sign-ins (accepted and denied), and log purges — filterable by action, actor, and outcome. | +| **Configuration** | The live config with **version history and rollback**. When a config store is configured the history is durable; changes and their audit entries are applied as one serialized operation, so a rollback cannot target the wrong version. | +| **API Keys** | Create, scope, and revoke API keys, and manage active dashboard sessions. | +| **Playground** | Send chat completions through the **real routing path** — the same targets, plugins, and strategy a production request takes. | +| **Tracing** | The request-trace view added in v1.4.0, pairing with the fullstack observability demo stack. | + +To see the console filled the way the marketing recording shows it, bring up the self-contained demo stack, which drives traffic through the gateway: + +```bash +make up-fullstack # gateway + Postgres + Jaeger + Prometheus + Grafana + mock upstream + load generator +# then open http://localhost:8080/ +``` + +## What it is not yet + +The dashboard is an honest read of the gateway, not a versioned control plane. Three limits are worth knowing before you lean on it: + +:::warning +- **It rides the unversioned `/admin/*` API.** A stable, versioned `/admin/v1` contract (Admin API v1) is future scope (targeted for v1.6.x), so treat these endpoints as internal — they can change between releases. +- **Provider status means *registration*, not *health*.** The Providers page tells you a provider is configured and whether its circuit is open, not that a live probe reached the upstream. +- **A `read_only` session can still invoke inference through the Playground.** A dedicated `inference:invoke` permission does not exist yet; a read-only key can send Playground requests, and each is billed by the upstream provider like any other completion. +::: + +## Related + +- [Authentication and API keys](/guides/auth) — scopes, `MASTER_KEY`, and the session model in depth +- [Request logging](/operations/request-logging) — the `duration_ms` / `ttft_ms` / `cost_usd` columns and `api_key_id` filtering the Logs page reads +- [Admin API reference](/api-reference/admin) — every `/admin/*` endpoint the console calls diff --git a/docs/guides/mcp.mdx b/docs/guides/mcp.mdx index ec1befe..a3ebb6a 100644 --- a/docs/guides/mcp.mdx +++ b/docs/guides/mcp.mdx @@ -1,10 +1,12 @@ --- title: MCP integration -description: "Connect MCP tool servers to the Ferro Labs AI Gateway for automatic agentic tool-calling with streaming support. The gateway runs the full tool loop — clients receive final text answers." -keywords: [Model Context Protocol, MCP gateway, MCP tool calling, agentic AI gateway, LLM tool use, MCP server integration, AI tool use, MCP streaming] +description: "Connect HTTP or stdio MCP servers to the AI Gateway for agentic tool-calling — the gateway runs the loop and re-checks guardrails every turn." +keywords: [Model Context Protocol, MCP gateway, MCP stdio transport, MCP tool calling, agentic AI gateway, LLM tool use, MCP server integration, MCP streaming] --- -Model Context Protocol (MCP) integration was added in **v0.8.0** and extended with streaming support in **v1.0.0**. When configured, the gateway connects to your MCP tool servers, injects available tools into chat completion requests, and runs the full agentic loop — so clients can receive a final text answer without implementing tool-calling logic themselves. +Model Context Protocol (MCP) integration connects the gateway to external tool servers. When `mcp_servers` is configured, the gateway advertises those servers' tools to the model and runs the full agentic loop itself — calling tools over MCP, feeding results back to the model, and repeating until a final answer comes back. Clients get a normal chat completion response; the tool round-trips are invisible. + +Two transports are supported: **Streamable HTTP** (`url`) for a running MCP endpoint, and **stdio** (`command` + `args`) for a subprocess the gateway launches and owns for its own lifetime — any `npx` or `uvx` MCP server works without standing up an HTTP endpoint for it. Each `mcp_servers` entry sets exactly one of `url` or `command`; setting both, or neither, is a config error naming the server. ## How it works @@ -15,32 +17,41 @@ sequenceDiagram participant MCP as MCP Tool Server participant LLM as AI Provider - Client->>Gateway: POST /v1/chat/completions - Gateway->>MCP: Fetch available tools (on startup) - Gateway->>LLM: Request + injected tools + Note over Gateway,MCP: On startup: initialize + discover tools (background) + Client->>Gateway: POST /v1/chat/completions (no tools in request) + Gateway->>LLM: Request + injected MCP tools LLM-->>Gateway: tool_calls response - loop Agentic loop (up to max_call_depth) + loop Agentic loop (up to max_call_depth, every turn re-runs guardrails/budget) Gateway->>MCP: Execute tool call MCP-->>Gateway: Tool result Gateway->>LLM: Append tool_result, request next step - LLM-->>Gateway: Final text response + LLM-->>Gateway: tool_calls or final response end Gateway-->>Client: Final text response ``` -The agentic loop runs inside the gateway. Your client sends a standard chat completion request and receives the final text answer. The intermediate tool calls are transparent. +Server initialization (the `initialize` + `tools/list` handshake) happens once in a background goroutine when the gateway starts, not per request — the gateway is ready to serve immediately, and MCP tool injection begins once that background init completes. Call `gateway.MCPInitDone()` to get a channel that closes when initialization finishes. + +:::note Tools are injected only when the caller sends none +MCP tools are added to a chat completion **only if the request's own `tools` array is empty**. If a client sends its own `tools`, the gateway leaves the request untouched — MCP does not participate, and standard client-side function calling works exactly as it would with MCP disabled. This avoids a turn where the model mixes one MCP-owned call with one caller-owned call that neither side can resolve. +::: ## Configuration -Add `mcp_servers` to your `config.yaml`: +Add `mcp_servers` to your `config.yaml`. A server entry uses either `url` (HTTP) or `command` (stdio): ```yaml mcp_servers: + # stdio transport — launched as a subprocess at gateway startup - name: filesystem - url: "http://localhost:3001/mcp" - timeout_seconds: 10 - max_call_depth: 3 - + command: npx + args: ["-y", "@modelcontextprotocol/server-filesystem", "/data"] + env: # the subprocess inherits NO gateway environment + SOME_TOKEN: "${SOME_TOKEN}" + timeout_seconds: 30 # per tool call; default 30 + required: false # default false; true gates /readyz on this server + + # Streamable HTTP transport — the gateway connects to a running endpoint - name: database url: "https://mcp-db.internal/mcp" headers: @@ -56,33 +67,39 @@ mcp_servers: | Field | Required | Default | Description | |---|---|---|---| -| `name` | Yes | — | Unique name for this MCP server | -| `url` | Yes | — | HTTP endpoint of the MCP server (Streamable HTTP transport) | -| `headers` | No | `{}` | HTTP headers to include (supports `${ENV_VAR}` interpolation) | -| `allowed_tools` | No | all tools | If set, only these tool names are injected and callable | -| `timeout_seconds` | No | `30` | Per-request timeout for calls to this server | -| `max_call_depth` | No | `5` | Maximum number of tool call rounds per request | +| `name` | Yes | — | Unique identifier for this server. Used in logs, metrics labels, and the `/readyz` body | +| `url` | Exactly one of `url` \| `command` | — | Streamable HTTP endpoint. Selects the HTTP transport | +| `command` | Exactly one of `url` \| `command` | — | Executable to launch as an MCP stdio server. Selects the stdio transport; the subprocess starts at gateway init and lives for the gateway's lifetime | +| `args` | No | `[]` | Command-line arguments passed to `command` (stdio only) | +| `env` | No | `{}` | Environment injected into the stdio subprocess (stdio only) — see [Subprocess environment isolation](#subprocess-environment-isolation) | +| `headers` | No | `{}` | HTTP headers sent on every MCP request (HTTP only). Supports `${VAR}` interpolation | +| `allowed_tools` | No | all tools | If set, only these tool names from this server are discovered and exposed to the model | +| `timeout_seconds` | No | `30` | Per-tool-call timeout for this server, both transports | +| `max_call_depth` | No | `5` | Bound on the agentic loop's turn depth. The **minimum positive value across all registered servers** is used | +| `required` | No | `false` | Makes this server's readiness a condition of `/readyz` — see [Readiness and required servers](#readiness-and-required-servers) | -## Startup behaviour +## Subprocess environment isolation -On `gateway.New()`, MCP connections are initialised in a background goroutine with a 60-second timeout. The gateway is ready to serve requests immediately — MCP tool injection begins once the background init completes. You can call `gateway.MCPInitDone()` to get a channel that closes when initialisation is finished. +A stdio MCP subprocess does **not** inherit the gateway's environment. It receives a minimal base — `PATH`, `HOME`, `LANG`, `TMPDIR` when set — plus exactly the keys listed in that server's `env`, which override the base. Gateway credentials such as `OPENAI_API_KEY` or `MASTER_KEY` never reach a subprocess implicitly. -## Authentication +This is isolation from *implicit* inheritance, not a prohibition: a value placed in `env` deliberately — including a credential, including one drawn from the gateway's own environment via `${VAR}` — is passed through as configured. Anything a server needs beyond the base four variables (`HTTPS_PROXY`, `NODE_PATH`, `SSL_CERT_FILE`, and on Windows `SYSTEMROOT`/`APPDATA`) must be listed explicitly. -MCP servers that require auth can receive credentials via the `headers` field. Environment variable interpolation (`${VAR}`) is supported so secrets are never hardcoded in config files: +`${VAR}` references in both `headers` and `env` are resolved when the MCP client is constructed, not when the config is loaded — so the config itself never stores the materialized secret, and it never appears in the config-history store or `GET /admin/config`. Only the braced form is a reference; a bare `$` is literal data, and an undefined variable is an error. ```yaml mcp_servers: - - name: secure-tools - url: "https://tools.internal/mcp" - headers: - Authorization: "Bearer ${MCP_TOOLS_TOKEN}" - X-Tenant-ID: "acme-corp" + - name: filesystem + command: npx + args: ["-y", "@modelcontextprotocol/server-filesystem", "/data"] + env: + SOME_TOKEN: "${SOME_TOKEN}" # resolved at client construction; ${…} only ``` +The subprocess's stderr is drained continuously and logged at debug level (the MCP spec treats stderr output as diagnostic, not an error) — this also prevents a full OS pipe buffer from blocking the child mid-write and stalling JSON-RPC. + ## Tool access control -Use `allowed_tools` to restrict which tools from an MCP server are exposed to the model. This is useful for read-only server access or capability scoping: +Use `allowed_tools` to restrict which tools from an MCP server are exposed to the model. Filtered tools are never discovered or advertised: ```yaml mcp_servers: @@ -94,74 +111,105 @@ mcp_servers: # write/delete tools from this server are NOT injected ``` -## Prompt injection risk +## Readiness and required servers -:::warning Security note -Allowing the model to execute arbitrary tool calls introduces risk. Always: -- Use `allowed_tools` to whitelist only the tools the model needs -- Prefer read-only tools where possible -- Set `max_call_depth` conservatively (3–5 is usually sufficient) -- Validate and sanitise all data before it reaches write-capable tools -::: +`mcp_servers[].required` (default `false`) makes one server's availability a condition of instance readiness. Every configured server's state is reported in the `GET /readyz` body regardless of `required`, so MCP health can be observed without gating on it: -## Compatible MCP servers +| `required` | Server unready | `/readyz` | +|---|---|---| +| absent / `false` | reported in the body | `200 ready` | +| `true` | reported in the body | `503`, reason `required mcp server unavailable` | -The gateway supports MCP servers that implement the **2025-11-25 Streamable HTTP transport**. Popular compatible servers include: +A server is **unready** when it never completed the `initialize` handshake — including one whose transport could not even be built (an unresolvable `${VAR}` in `headers` or `env`). Death **after** a successful handshake is detected for **stdio servers only**: a crashed subprocess is noticed (its stderr pipe closing, confirmed with an MCP ping before anything is withdrawn) and its tools are withdrawn from the model. An HTTP server that becomes unreachable after a successful handshake is **not currently detected** — it keeps reporting ready with its tools advertised, and calls to it fail per request. Don't rely on `required: true` to pull an instance out of rotation when an HTTP MCP server goes down. -- [Filesystem MCP server](https://github.com/modelcontextprotocol/servers) — read/list/write files -- [Postgres MCP server](https://github.com/modelcontextprotocol/servers) — SQL query execution -- Any server implementing the MCP 2025-11-25 spec with HTTP transport +The failure *reason* is deliberately omitted from the unauthenticated `/readyz` body, since it can quote a server URL, an authorization header, or a subprocess command line. It's logged server-side and served on the bearer-authenticated `GET /admin/health` instead (`read_only` or `admin` scope). -## Testing the connection +Set `required: true` only for a server the deployment genuinely cannot serve without — a required server that's down stops **all** traffic through the instance, including requests that use no tools at all. -After starting the gateway with `mcp_servers` configured, verify tools are loaded by checking the startup logs — the gateway logs MCP tool discovery once background initialisation completes: +## Agentic loop mechanics -```bash -# The gateway logs MCP tool discovery on startup -docker logs ferrogw 2>&1 | grep mcp -``` +Once MCP is active for a request, the executor loops until the model stops requesting tool calls or `max_call_depth` is reached: -## Streaming requests +- **Ownership.** A tool call is only executed if the gateway's registry owns that tool name. A choice mixing an MCP-owned call with a caller-owned call is never half-executed — it's handed back to the client whole, since the provider would reject an unmatched `tool_call_id`. +- **Depth limit.** At the depth limit, pending MCP-owned tool calls are dropped and the response finishes with `finish_reason: "length"` rather than asking the client to satisfy tools it never declared. +- **Guardrails run on every turn.** Every turn of the loop passes through the same `before_request` plugins (guardrails, rate limiting, budget) that gate the initial provider call — not just the first turn. Tool **results** are content the caller never wrote and the operator has the least reason to trust, so a guardrail can reject a request mid-loop, and a budget check can stop an overspending loop partway through rather than after it completes. +- **Failures reach the model as generic messages** (`timed out`, `server unavailable`, `the tool call failed`) — the real error, which can quote URLs or command lines, is logged server-side only. -Since **v1.0.0-rc.1**, clients may send `stream: true` when MCP servers are configured. The gateway transparently redirects streaming requests through the full agentic loop — all tool calls are resolved inside the gateway, and the final text answer is returned as a single-chunk stream response. Clients receive correct SSE output without needing to handle intermediate tool-call messages. +:::warning Security note +Allowing the model to execute tool calls introduces risk, and tool **results** are the least-trusted content in the loop — an external MCP server, not the caller, produced them. Guardrails now run every turn for this reason. Additionally: +- Use `allowed_tools` to expose only the tools the model needs +- Prefer read-only tools where possible +- Set `max_call_depth` conservatively (3–5 is usually sufficient) +- Validate and sanitise all data before it reaches write-capable tools +::: -```python -# stream: true works — final answer is delivered via SSE after the tool loop completes -response = client.chat.completions.create( - model="claude-3-5-sonnet-20241022", - stream=True, - messages=[{"role": "user", "content": "List the failing tests in gateway_test.go"}], -) -for chunk in response: - print(chunk.choices[0].delta.content or "", end="") -``` +## Observability -## Example: filesystem tools +MCP exposes dedicated Prometheus series on `/metrics`: -Start a local filesystem MCP server and connect it to the gateway: +| Metric | Labels | Description | +|---|---|---| +| `gateway_mcp_server_up` | `server_name` | `1` when a server completed its handshake and its transport is alive, `0` otherwise | +| `gateway_mcp_server_init_failures_total` | `server_name` | Incremented each time a server fails to initialize | +| `ferrogw_mcp_tool_calls_total` | `server_name`, `tool_name`, `status` | Total MCP tool calls (`status` is `ok` or `error`) | +| `ferrogw_mcp_tool_call_duration_seconds` | `server_name`, `tool_name` | Latency histogram for individual tool calls | +| `ferrogw_mcp_unknown_tool_calls_total` | `tool_name` | Tool calls the model requested for a name no registered server advertises | -```bash -# 1. Start the MCP filesystem server -npx @modelcontextprotocol/server-filesystem /path/to/workspace +## Example: local filesystem tools via stdio -# 2. config.yaml +Point the gateway at a filesystem MCP server with no separate process to manage — the gateway launches it: + +```yaml mcp_servers: - name: filesystem - url: "http://localhost:3000/mcp" + command: npx + args: ["-y", "@modelcontextprotocol/server-filesystem", "/path/to/workspace"] allowed_tools: [read_file, list_directory, search_files] max_call_depth: 4 ``` -Then ask the model a question that requires reading files: +Then ask the model a question that requires reading files. Send no `tools` array so MCP participates: ```python response = client.chat.completions.create( - model="claude-3-5-sonnet-20241022", + model="claude-sonnet-4-5", messages=[{ "role": "user", "content": "What tests are failing in the src/gateway_test.go file?" }], ) -# The gateway reads the file via MCP, sends content to Claude, returns the answer +# The gateway reads the file via MCP, sends content to the model, returns the answer print(response.choices[0].message.content) ``` + +## Example: remote HTTP server with auth + +```yaml +mcp_servers: + - name: secure-tools + url: "https://tools.internal/mcp" + headers: + Authorization: "Bearer ${MCP_TOOLS_TOKEN}" + X-Tenant-ID: "acme-corp" +``` + +## Streaming requests + +Clients may send `stream: true` when MCP is active. The gateway diverts streaming requests through the full (non-streaming) agentic loop internally — every tool call is resolved inside the gateway — then delivers the final answer as a single SSE chunk once the loop completes. Clients receive correct SSE output without handling intermediate tool-call messages, but the response is not token-by-token for MCP-driven turns: + +```python +response = client.chat.completions.create( + model="claude-sonnet-4-5", + stream=True, + messages=[{"role": "user", "content": "List the failing tests in gateway_test.go"}], +) +for chunk in response: + print(chunk.choices[0].delta.content or "", end="") +``` + +## Related + +- [Configuration reference](/getting-started/configuration) +- [Plugins overview](/plugins) +- [Monitoring](/operations/monitoring) +- [Request lifecycle](/getting-started/request-lifecycle) diff --git a/docs/guides/migration-litellm.mdx b/docs/guides/migration-litellm.mdx index cce8830..3ee58ad 100644 --- a/docs/guides/migration-litellm.mdx +++ b/docs/guides/migration-litellm.mdx @@ -1,6 +1,6 @@ --- title: Migrate from LiteLLM -description: Step-by-step guide to replace LiteLLM with the Ferro Labs AI Gateway — swap the base URL and API key, convert provider configs, and retain existing retry and fallback logic. +description: "Replace LiteLLM with the Ferro Labs AI Gateway — swap the base URL and API key, convert provider configs, and keep your retry and fallback logic." keywords: [LiteLLM migration, LiteLLM alternative, LiteLLM vs Ferro Labs Labs AI Gateway, migrate from LiteLLM, LLM proxy migration] --- diff --git a/docs/guides/migration-openrouter.mdx b/docs/guides/migration-openrouter.mdx index 8b0f691..9739854 100644 --- a/docs/guides/migration-openrouter.mdx +++ b/docs/guides/migration-openrouter.mdx @@ -98,7 +98,7 @@ response = client.chat.completions.create( # After (Ferro) — plain model ID, provider chosen by routing config response = client.chat.completions.create( model="gpt-4o", - messages=[{"role": "user", "content": "Hello from Ferro AI Gateway"}], + messages=[{"role": "user", "content": "Hello from Ferro Labs AI Gateway"}], ) ``` @@ -211,5 +211,5 @@ plugins: - **OpenRouter is hosted; Ferro is self-hosted.** You bring your own provider keys and run the gateway yourself. See [getting started](/getting-started/quickstart) for Docker and binary options. - **`provider/model` slugs are not supported.** Pass plain model IDs and let `targets`/`strategy` choose the provider, or define `aliases`. This is the only required code change for most apps. -- **Automatic provider selection** in OpenRouter becomes explicit routing in Ferro — `fallback`, `loadbalance`, `conditional`, and more. See [Routing policies](/guides/routing-policies). +- **Automatic provider selection** in OpenRouter becomes explicit routing in Ferro — `fallback`, `loadbalance`, `conditional`, and more. See [Routing policies](/routing). - **OpenRouter dashboard & analytics**: The OSS gateway exposes structured logs via the `request-logger` plugin and a queryable admin API. Hosted dashboards and durable analytics are in [Ferro Labs Managed](https://ferrolabs.ai). See [OSS vs Ferro Labs Managed](/guides/oss-vs-ferrocloud). diff --git a/docs/guides/observability.mdx b/docs/guides/observability.mdx index 0ba9356..d1b9b6c 100644 --- a/docs/guides/observability.mdx +++ b/docs/guides/observability.mdx @@ -1,43 +1,80 @@ --- title: Observability -description: Full observability for your LLM traffic — Prometheus metrics, structured JSON logs with trace IDs, and a deep per-provider health endpoint in the Ferro Labs AI Gateway. -keywords: [LLM observability, AI gateway metrics, Prometheus LLM, AI gateway monitoring, structured logging AI, LLM tracing] +description: Prometheus metrics behind a scoped bearer token, OpenTelemetry tracing over OTLP, trace-ID-unified structured logs, and /livez, /readyz, /health probes. +keywords: [LLM observability, AI gateway metrics, Prometheus LLM, AI gateway monitoring, structured logging AI, LLM tracing, readyz livez health check] --- -The gateway ships with four observability layers: Prometheus metrics, OpenTelemetry tracing, structured log output, and a deep health endpoint. +The gateway ships four observability layers: Prometheus metrics, OpenTelemetry tracing, structured JSON logs, and a set of liveness/readiness/health probes. ## Prometheus metrics -Metrics are exposed at `GET /metrics` in the standard Prometheus text format. Scrape this endpoint with your Prometheus server. All metric names use the `gateway_` prefix. +Metrics are exposed at `GET /metrics` in the standard Prometheus text format, mounted via `promhttp` on the default registry. All metric names use the `gateway_` prefix (MCP tool-call metrics are the one exception — see below). + +:::danger /metrics requires a bearer token +`/metrics` is **not** an open endpoint. It sits behind the same auth chain as the admin API and requires a bearer token — `MASTER_KEY`, an issued `fgw_` API key, or a dashboard session — carrying the `read_only` or `admin` scope. An unauthenticated scrape gets `401`. Every scrape config must supply the token. +::: + +Issue a dedicated `read_only` key for your scraper (scopes default to `read_only` when omitted — see [Authentication](/guides/auth)): + +```bash +curl -X POST http://localhost:8080/admin/keys \ + -H "Authorization: Bearer $MASTER_KEY" \ + -H "Content-Type: application/json" \ + -d '{"name": "monitoring-scraper"}' +``` ### Available metrics | Metric | Type | Labels | Description | |---|---|---|---| -| `gateway_requests_total` | Counter | `provider`, `model`, `status` | Total requests processed (`status`: `success`\|`error`\|`rejected`) | -| `gateway_request_duration_seconds` | Histogram | `provider`, `model` | End-to-end request latency | +| `gateway_requests_total` | Counter | `provider`, `model`, `status` | Completed requests; `status` is `success`\|`error`\|`rejected`. `provider` is `none` when no provider was chosen and `cache` on a response-cache hit; `model` is `unknown` for a client-supplied model no target serves | +| `gateway_request_duration_seconds` | Histogram | `provider`, `model` | End-to-end request latency (successes and failures alike); buckets .005s–30s | | `gateway_tokens_input_total` | Counter | `provider`, `model` | Prompt tokens sent to providers | | `gateway_tokens_output_total` | Counter | `provider`, `model` | Completion tokens received from providers | -| `gateway_provider_errors_total` | Counter | `provider`, `error_type` | Provider errors by type (`provider_error`\|`circuit_open`\|`timeout`) | -| `gateway_circuit_breaker_state` | Gauge | `provider` | Circuit breaker state (`0`=closed, `1`=open, `2`=half-open) | -| `gateway_rate_limit_rejections_total` | Counter | `key_type` | Requests rejected by rate limiting (`key_type`: `ip`\|`api_key`\|`plugin`) | -| `gateway_request_cost_usd_total` | Counter | `provider`, `model` | Estimated cumulative request cost in USD (public pricing tables) | -| `gateway_server_connections_current` | Gauge | `state` | Current inbound HTTP connections by state | +| `gateway_request_cost_usd_total` | Counter | `provider`, `model` | Estimated cumulative cost in USD from public pricing tables | +| `gateway_provider_errors_total` | Counter | `provider`, `error_type` | Errors by type: `provider_error`, `circuit_open`, `timeout`, `client_canceled`, `backpressure`, `plugin_error`. Alert only on `provider_error` and `timeout` — the rest are the gateway declining or shedding load, or the caller leaving | +| `gateway_provider_init_failures_total` | Counter | `provider` | Providers whose factory failed at startup (warned, then skipped) — the only machine-readable signal a configured provider never came up | +| `gateway_circuit_breaker_state` | Gauge | `provider` | Circuit state per target, resolved from the live breaker at scrape time: `0`=closed, `1`=open, `2`=half-open. A series exists only for a target that has a breaker configured — absent means none is configured, not "never tripped" | +| `gateway_mcp_server_up` | Gauge | `server_name` | MCP server availability: `0`=not ready, `1`=ready and advertising tools. A drop from `1` to `0` with no config change means the transport died (e.g. a stdio subprocess exited) | +| `gateway_mcp_server_init_failures_total` | Counter | `server_name` | MCP servers whose initialize handshake or tool discovery failed | +| `gateway_rate_limit_rejections_total` | Counter | `key_type` | Requests rejected by rate limiting (`key_type`: `ip` — per-IP middleware; `admin_session` — the sign-in route's own limiter; `plugin` — the rate-limit plugin's global/per-key/per-user buckets) | +| `gateway_server_connections_current` | Gauge | `state` | Current inbound HTTP connections by state (`active`\|`idle`) | | `gateway_server_connection_transitions_total` | Counter | `state` | Inbound HTTP connection state transitions | -| `gateway_hook_events_dropped_total` | Counter | `subject` | Hook dispatches dropped because the worker queue was full | +| `gateway_hook_events_dropped_total` | Counter | `subject` | Hook dispatches dropped because the hook worker queue was full | +| `gateway_observability_events_dropped_total` | Counter | `subject` | Observability events (`gateway.request.completed`/`failed`) dropped because the exporter dispatch queue was full — `RecordEvent` is non-blocking | | `gateway_catalog_loads_total` | Counter | `source`, `result` | Model-catalog load attempts (`source`: `remote`\|`fallback`; `result`: `success`\|`error`) | +MCP **tool-call** metrics use the `ferrogw_mcp_` prefix instead of `gateway_` — they're registered separately in the MCP executor: + +| Metric | Type | Labels | Description | +|---|---|---|---| +| `ferrogw_mcp_tool_calls_total` | Counter | `server_name`, `tool_name`, `status` | MCP tool calls made (`status`: `ok`\|`error`) | +| `ferrogw_mcp_tool_call_duration_seconds` | Histogram | `server_name`, `tool_name` | Latency of individual MCP tool calls | +| `ferrogw_mcp_unknown_tool_calls_total` | Counter | `tool_name` | Tool calls naming a tool no registered MCP server advertises (a hallucinated tool name) | + ### Example Prometheus scrape config +Prometheus's `authorization` block sends the bearer token on every scrape: + ```yaml scrape_configs: - job_name: ferrogw + metrics_path: /metrics + authorization: + type: Bearer + credentials: "fgw_your_read_only_scrape_key" + # or: credentials_file: /etc/prometheus/secrets/ferrogw-metrics-token static_configs: - targets: ["localhost:8080"] - metrics_path: /metrics scrape_interval: 15s ``` +A manual check: + +```bash +curl -H "Authorization: Bearer $FERROGW_METRICS_TOKEN" http://localhost:8080/metrics +``` + ### Useful PromQL queries ```promql @@ -47,7 +84,10 @@ rate(gateway_requests_total[5m]) # P99 request latency histogram_quantile(0.99, rate(gateway_request_duration_seconds_bucket[5m])) -# Error rate percentage (failed requests over all requests) +# Error rate percentage — status="rejected" (denied by a guardrail/budget +# plugin) is deliberately excluded from the numerator: it's a policy +# decision, not a gateway or provider fault, and folding it in inflates the +# "error" signal with traffic the gateway handled correctly. sum(rate(gateway_requests_total{status="error"}[5m])) / sum(rate(gateway_requests_total[5m])) * 100 @@ -59,6 +99,9 @@ rate(gateway_request_cost_usd_total[5m]) * 3600 # Open circuit breakers gateway_circuit_breaker_state == 1 + +# MCP servers that dropped out of rotation +gateway_mcp_server_up == 0 ``` ## OpenTelemetry tracing @@ -69,7 +112,9 @@ Each request opens a `gateway.request` root span stamped with `gen_ai.*` and `fe ### Unified trace ID -The OTel `trace_id`, the structured-log `trace_id`, and the `X-Request-ID` response header are all the **same value**. Copy a trace ID from a log line and look it up directly in your tracing backend. +The OTel `trace_id`, the structured-log `trace_id`, the `ferro.gateway.trace_id` span attribute, and the `X-Request-ID` response header are all the **same value**: a 32-character lowercase hex string (16 raw bytes), not a dashed UUID. Copy a trace ID from a log line and look it up directly in your tracing backend. + +An inbound `X-Request-ID` (or a W3C `traceparent`) is adopted only when it can *be* a trace ID — 32 hex characters, not all-zero. A client-supplied value that isn't (a UUID, a slug, an upstream proxy's opaque request ID) is replaced with a freshly generated one rather than echoed back, so the response header, the logs, and the OTel trace never disagree on the ID for one request. ### Configuration @@ -79,7 +124,7 @@ Configure tracing under the `observability` block in your gateway config: observability: tracing: enabled: true - endpoint: "" # host:port; blank falls back to OTEL_EXPORTER_OTLP_ENDPOINT + endpoint: "" # host:port or URL; blank falls back to OTEL_EXPORTER_OTLP_* env protocol: grpc # grpc | http/protobuf (https:// endpoint ⇒ TLS, else insecure) service_name: ferrogw sample_ratio: 1.0 # head sampler, 0.0–1.0 @@ -91,19 +136,24 @@ observability: config: {} ``` -Standard OpenTelemetry environment variables take precedence over the config file: +`tracing.enabled` is tri-state: omit it and the gateway infers tracing from whether an endpoint or exporter is configured; set it `false` to force tracing off even with an endpoint present; set it `true` to force it on. + +The gateway itself reads exactly **two** `OTEL_*` environment variables, and either one alone turns tracing on: | Variable | Purpose | |---|---| -| `OTEL_EXPORTER_OTLP_ENDPOINT` | OTLP collector endpoint; enables tracing when set | -| `OTEL_EXPORTER_OTLP_HEADERS` | Headers sent to the collector (e.g. auth tokens) | -| `OTEL_TRACES_SAMPLER` / `OTEL_TRACES_SAMPLER_ARG` | Head-sampler overrides | +| `OTEL_EXPORTER_OTLP_ENDPOINT` | OTLP base endpoint; outranks `observability.tracing.endpoint` | +| `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` | Signal-specific traces endpoint, used verbatim; outranks the variable above | + +:::note OTEL_TRACES_SAMPLER has no effect +The head sampler comes **only** from `observability.tracing.sample_ratio` and is wrapped in a `ParentBased` sampler, so an inbound sampled `traceparent` is always followed regardless of the ratio. `OTEL_TRACES_SAMPLER` / `OTEL_TRACES_SAMPLER_ARG` are standard OTel SDK variables the gateway never reads — setting them silently does nothing. Every other tracing setting (protocol, service name, privacy level, shutdown grace, headers) comes from config only; only the endpoint has an environment override. +::: -When both a config value and the matching `OTEL_*` variable are present, the environment variable wins. +`OTEL_EXPORTER_OTLP_HEADERS` is honored too, but indirectly: it reaches the OTLP SDK's own transport, not gateway code, so it layers underneath whatever `observability.tracing.headers` resolves to. ### Export headers -To attach static metadata or backend auth tokens to every OTLP export, set the in-config `observability.tracing.headers` map. Each value supports `${ENV_VAR}` (and `$ENV_VAR`) interpolation that is resolved at exporter-build time — so secrets live in the environment, never literally in the in-memory config. A header whose value resolves to empty (e.g. it references an unset variable) is dropped and a warning is logged. +To attach static metadata or backend auth tokens to every OTLP export, set the in-config `observability.tracing.headers` map: ```yaml observability: @@ -116,58 +166,61 @@ observability: x-ferro-env: production # literal value, passed through ``` -The standard `OTEL_EXPORTER_OTLP_HEADERS` environment variable is also honored and takes precedence. +`${VAR}` is the **only** reference form — a bare `$` (`$100`, `pa$$w0rd`) is always literal data, never expanded. This differs from the general `${VAR}` rule used elsewhere in the config (plugin `config`, MCP `headers`/`env`, `observability.exporters[].config`), where an undefined variable is a hard construction-time error: for `tracing.headers` specifically, a header whose reference is undefined — or that resolves to an empty string — is instead **dropped with a logged warning**, and every other header still gets sent. Tracing is auxiliary; one mistyped trace header shouldn't take the rest of the export down with it. ### Span attributes Each span is stamped with two attribute groups. Group A follows the [OpenTelemetry GenAI semantic conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/); Group B is the Ferro `ferro.*` extension namespace. Every build advertises its attribute schema version via `ferro.schema.version` (currently **`1.0.0-draft`**), so exporters can branch on schema migrations. +Not every declared constant is wired into a live span yet — the **Status** column below tells you which ones are. A `Planned` name is stable (safe to reference in a dashboard you're building ahead of time) but won't appear on a span until a later release emits it. + **Group A — `gen_ai.*` (OpenTelemetry GenAI conventions)** -| Attribute | Meaning | -|---|---| -| `gen_ai.system` | Provider system (e.g. `openai`, `anthropic`) | -| `gen_ai.operation.name` | Operation (e.g. `chat`) | -| `gen_ai.request.model` | Requested model ID | -| `gen_ai.response.model` | Model that actually served the response | -| `gen_ai.request.max_tokens` | Requested max output tokens | -| `gen_ai.request.temperature` | Sampling temperature | -| `gen_ai.request.top_p` | Nucleus-sampling top-p | -| `gen_ai.request.is_stream` | Whether streaming was requested | -| `gen_ai.usage.input_tokens` | Prompt tokens | -| `gen_ai.usage.output_tokens` | Completion tokens | -| `gen_ai.usage.reasoning_tokens` | Reasoning tokens (reasoning models) | -| `gen_ai.response.finish_reasons` | Finish reasons | +| Attribute | Status | Meaning | +|---|---|---| +| `gen_ai.system` | Emitted | Provider system (e.g. `openai`, `anthropic`) | +| `gen_ai.operation.name` | Emitted | Operation (e.g. `chat`) | +| `gen_ai.request.model` | Emitted | Requested model ID | +| `gen_ai.response.model` | Emitted | Model that actually served the response | +| `gen_ai.request.is_stream` | Emitted | Whether streaming was requested | +| `gen_ai.usage.input_tokens` | Emitted | Prompt tokens | +| `gen_ai.usage.output_tokens` | Emitted | Completion tokens | +| `gen_ai.usage.reasoning_tokens` | Emitted | Reasoning tokens (reasoning models) | +| `gen_ai.request.max_tokens` | Planned | Requested max output tokens | +| `gen_ai.request.temperature` | Planned | Sampling temperature | +| `gen_ai.request.top_p` | Planned | Nucleus-sampling top-p | +| `gen_ai.response.finish_reasons` | Planned | Finish reasons | **Group B — `ferro.*` (Ferro extension)** -| Attribute | Meaning | -|---|---| -| `ferro.schema.version` | Attribute schema version (`1.0.0-draft`) | -| `ferro.gateway.trace_id` | Unified trace ID (same value as the log `trace_id` / `X-Request-ID`) | -| `ferro.gateway.version` | Gateway build version | -| `ferro.routing.strategy` | Routing strategy used (`fallback`, `loadbalance`, …) | -| `ferro.routing.target_key` | Selected target / virtual key | -| `ferro.routing.attempt` | Attempt number within the strategy | -| `ferro.routing.ab_variant_label` | A/B variant label, when applicable | -| `ferro.cost.usd` | Estimated total request cost in USD | -| `ferro.cost.input_usd` / `ferro.cost.output_usd` | Estimated input / output cost | -| `ferro.cost.cache_read_usd` / `ferro.cost.cache_write_usd` | Estimated prompt-cache read / write cost | -| `ferro.cost.reasoning_usd` | Estimated reasoning-token cost | -| `ferro.cost.model_found` | Whether the model was found in the catalog for pricing | -| `ferro.cache.hit` | Whether a response-cache hit served the request | -| `ferro.cache.kind` | Cache kind | -| `ferro.plugin.name` / `ferro.plugin.kind` | Plugin identity (child spans) | -| `ferro.plugin.stage` | Plugin stage (`before_request`, `after_request`, `on_error`) | -| `ferro.plugin.outcome` / `ferro.plugin.reason` | Plugin outcome and reason | -| `ferro.mcp.server` / `ferro.mcp.tool` | MCP server and tool name (child spans) | -| `ferro.mcp.depth` / `ferro.mcp.latency_ms` | MCP call depth and latency | -| `ferro.stream.time_to_first_token_ms` | Streaming time-to-first-token | -| `ferro.stream.time_to_last_token_ms` | Streaming time-to-last-token | -| `ferro.circuit_breaker.state` / `ferro.circuit_breaker.opened` | Circuit-breaker state and whether it opened during the request | -| `ferro.request.api_key_id` / `ferro.request.tenant_id` | Request API key ID and tenant ID | -| `ferro.error.upstream_status` | Upstream HTTP status on failure | -| `ferro.error.retry_count` | Number of retries performed | +| Attribute | Status | Meaning | +|---|---|---| +| `ferro.schema.version` | Emitted | Attribute schema version (`1.0.0-draft`) | +| `ferro.gateway.trace_id` | Emitted | Unified trace ID (same value as the log `trace_id` / `X-Request-ID`) | +| `ferro.routing.strategy` | Emitted | Routing strategy used (`fallback`, `loadbalance`, …) | +| `ferro.routing.target_key` | Emitted | Selected target / virtual key | +| `ferro.cost.usd` | Emitted | Estimated total request cost in USD | +| `ferro.cost.input_usd` / `ferro.cost.output_usd` | Emitted | Estimated input / output cost | +| `ferro.cost.cache_read_usd` / `ferro.cost.cache_write_usd` | Emitted | Estimated prompt-cache read / write cost | +| `ferro.cost.reasoning_usd` | Emitted | Estimated reasoning-token cost | +| `ferro.cost.model_found` | Emitted | Whether the model was found in the catalog for pricing | +| `ferro.plugin.name` / `ferro.plugin.kind` | Emitted | Plugin identity (child spans) | +| `ferro.plugin.stage` | Emitted | Plugin stage (`before_request`, `after_request`, `on_error`) | +| `ferro.plugin.outcome` / `ferro.plugin.reason` | Emitted | Plugin outcome (`ok`\|`rejected`\|`error`) and reason | +| `ferro.mcp.server` / `ferro.mcp.tool` | Emitted | MCP server and tool name (child spans) | +| `ferro.mcp.latency_ms` | Emitted | MCP call latency | +| `ferro.stream.time_to_first_token_ms` | Emitted | Streaming time-to-first-token | +| `ferro.stream.time_to_last_token_ms` | Emitted | Streaming time-to-last-token | +| `ferro.gateway.version` | Planned | Gateway build version | +| `ferro.routing.attempt` | Planned | Attempt number within the strategy | +| `ferro.routing.ab_variant_label` | Planned | A/B variant label — not yet on the span; today the A/B variant only shows up in `DEBUG`-level logs | +| `ferro.cache.hit` / `ferro.cache.kind` | Planned | Response-cache hit and cache kind | +| `ferro.mcp.depth` | Planned | MCP call depth | +| `ferro.circuit_breaker.state` / `ferro.circuit_breaker.opened` | Planned | Circuit-breaker state and whether it opened during the request | +| `ferro.request.api_key_id` / `ferro.request.tenant_id` | Planned | Request API key ID and tenant ID | +| `ferro.error.upstream_status` | Planned | Upstream HTTP status on failure | +| `ferro.error.retry_count` | Planned | Number of retries performed | +| `ferro.forwarded_params` | Planned | Sanitized names (never values) of parameters forwarded to the provider | ### Privacy levels @@ -175,72 +228,146 @@ Each span is stamped with two attribute groups. Group A follows the [OpenTelemet | Level | Behavior | |---|---| -| `none` | No prompt/response content; error messages omitted | -| `metadata` | **Default.** Metadata only; error messages redacted (email, JWT, AWS keys) | +| `none` | A static `"redacted"` string only — no error text | +| `metadata` | **Default.** Error messages redacted (email, JWT, AWS keys tokenised) | | `full` | Raw error text included — use only in trusted environments | +No prompt or response content is exported at any privacy level. + ### Exporters -Beyond raw OTLP, the gateway exposes an exporter event seam: registered exporters receive `gateway.request.completed` / `gateway.request.failed` events for bridging to backends like LangSmith, Langfuse, or Datadog. No exporters ship in the core gateway — they live in the separate [`ai-gateway-plugins`](https://github.com/ferro-labs/ai-gateway) repo. List enabled exporters under `observability.exporters`; unknown or failed exporters are logged and skipped (non-fatal). +Beyond raw OTLP, the gateway exposes an exporter event seam: registered exporters receive `gateway.request.completed` / `gateway.request.failed` events for bridging to backends like LangSmith or Langfuse. No exporters ship in the core gateway — they live in the separate [`ai-gateway-plugins`](https://github.com/ferro-labs/ai-gateway-plugins) repo. List enabled exporters under `observability.exporters`; an unregistered `name`, or one whose `config` carries an undefined `${VAR}` reference, is logged and skipped without stopping the gateway (non-fatal) — the rest of the exporters and the OTLP pipeline keep running. + +To see traces end to end without wiring your own collector, run the bundled demo stack — gateway, Prometheus, Grafana, Jaeger, and a traffic generator — with `make up-fullstack`, then open the dashboard's **Tracing** page. See [The embedded dashboard](/guides/dashboard). ## Structured JSON logs -The gateway writes structured JSON to stdout. Each log line includes: +The gateway writes structured JSON to stdout (`log/slog`'s JSON handler) — there's no plain-text mode. `LOG_LEVEL` (`debug`\|`info`\|`warn`\|`error`, default `info`) is the only knob: -| Field | Description | -|---|---| -| `time` | ISO 8601 timestamp | -| `level` | `debug`, `info`, `warn`, `error` | -| `trace_id` | Per-request UUID for log correlation | -| `msg` | Log message | -| `provider` | Provider name (on request/response lines) | -| `model` | Model ID | -| `latency_ms` | Provider round-trip latency in milliseconds | -| `status` | HTTP status code | -| `tokens_prompt` | Prompt token count | -| `tokens_completion` | Completion token count | - -Example log line: +```bash +export LOG_LEVEL=debug +``` + +Every request produces an **access log line** — `msg: "http request"` — regardless of level, once the response finishes: ```json { - "time": "2026-03-11T10:23:45Z", - "level": "info", - "trace_id": "a3f9b1c2-d4e5-4678-8901-abcdef012345", - "msg": "request complete", - "provider": "anthropic", - "model": "claude-3-5-sonnet-20241022", - "latency_ms": 412, + "time": "2026-08-06T10:23:45.128Z", + "level": "INFO", + "msg": "http request", + "trace_id": "4bf92f3577b34da6a3ce929d0e0e4736", + "method": "POST", + "path": "/v1/chat/completions", "status": 200, - "tokens_prompt": 312, - "tokens_completion": 87 + "bytes": 842, + "duration_ms": 412, + "remote": "10.0.4.12:51322" } ``` -### Log level +A `4xx` status logs the line at `warn`, a `5xx` at `error`; every field is HTTP-level (method, path, status, byte count, duration, client address) and complements the request-logger **plugin**, which records LLM semantics (model, tokens, cost) — see [Request logging](/operations/request-logging). -Set the log level with `LOG_LEVEL` (default: `info`). Use `LOG_FORMAT=text` for human-readable output during development. +At `LOG_LEVEL=debug`, a successful route additionally logs a richer completion line carrying the resolved provider, token counts, and cost: -```bash -export LOG_LEVEL=debug -export LOG_FORMAT=text +```json +{ + "time": "2026-08-06T10:23:45.127Z", + "level": "DEBUG", + "msg": "request completed", + "trace_id": "4bf92f3577b34da6a3ce929d0e0e4736", + "model": "gpt-4o", + "provider": "openai", + "latency_ms": 412, + "tokens_in": 312, + "tokens_out": 87, + "cost_usd": 0.0041 +} +``` + +A failed route logs a `request failed` line at `error` level unconditionally (no `LOG_LEVEL=debug` required): + +```json +{ + "time": "2026-08-06T10:23:46.003Z", + "level": "ERROR", + "msg": "request failed", + "trace_id": "4bf92f3577b34da6a3ce929d0e0e4736", + "model": "gpt-4o", + "latency_ms": 875, + "error": "provider_error: upstream returned 503" +} +``` + +Every string field and every logged error passes through a redaction filter before it reaches stdout, so a raw upstream error carrying a configured credential is scrubbed at the log sink itself, not just at the call sites that remember to redact. + +## Liveness, readiness, and health + +The gateway splits "is the process up" from "can it serve traffic" from "give me a diagnostic dump" across three unauthenticated endpoints: + +| Endpoint | Answers | On failure | +|---|---|---| +| `GET /livez` | Is the process alive? No dependency checks — always `200 {"status":"ok"}`. Orchestrators use it to decide whether to **restart**. | never fails | +| `GET /readyz` | Can this instance serve traffic right now? Gates on config load, backing-store reachability, and **target routability**. Orchestrators use it to decide whether to **route** traffic here. | `503` | +| `GET /health` | Deep diagnostic: every registered provider's status, circuit state, and model count. | `503` only when zero providers are registered at all | + +### `/readyz`: target routability + +A target is **routable** when a provider is registered under its `virtual_key` (the credential env var is set) and that provider's circuit isn't open. `/readyz` is ready when **at least one** configured target is routable — not all of them, since one dead target among several is a fallback/load-balance case, not an outage: + +```json title="200 — ready" +{ + "status": "ready", + "providers": [ + { "name": "openai", "circuit": "closed" }, + { "name": "anthropic", "circuit": "closed" } + ], + "targets": [ + { "name": "openai", "routable": true }, + { "name": "anthropic", "routable": true } + ], + "mcp_servers": [ + { "name": "filesystem", "ready": true, "required": false } + ] +} ``` -## Health endpoint +`mcp_servers` is present only when `mcp_servers[]` is configured. Reason strings are fixed (the endpoint is unauthenticated, so nothing sensitive — a DSN, a host, a credential — ever appears in the body): -`GET /health` returns a deep health check with per-provider availability and latency: +```json title="503 — not ready" +{ "status": "not_ready", "reason": "no routable targets" } +``` + +| Reason | Cause | +|---|---| +| `no routable targets` | Zero configured targets are routable — every credentialed provider is either unnamed by any target or has its circuit open | +| `required mcp server unavailable` | An `mcp_servers[]` entry with `required: true` hasn't completed its initialize handshake — see [MCP integration](/guides/mcp) | +| `store unreachable` | The key store or config manager failed a reachability ping | +| `gateway not configured` | No gateway instance is wired to the server yet | + +Every server's state is reported under `mcp_servers` whether or not it's `required`, so MCP health is observable without gating readiness on an optional server. The failure detail behind a bad MCP server (a URL, an auth header, a subprocess command line) is deliberately not in this unauthenticated body — it's logged server-side. + +### `/health`: deep diagnostic + +`/health` lists every **registered** provider (not every configured target) with its circuit state and model count: ```json { "status": "ok", - "providers": { - "openai": { "healthy": true, "latency_ms": 245 }, - "anthropic": { "healthy": true, "latency_ms": 312 }, - "groq": { "healthy": false, "error": "connection refused" } - } + "providers": [ + { "name": "openai", "status": "available", "circuit": "closed", "models": 42 }, + { "name": "anthropic", "status": "available", "circuit": "open", "models": 18 } + ] } ``` -Returns `200 OK` if at least one provider is healthy, `503 Service Unavailable` if all providers are down. +`status` is `"ok"` whenever at least one provider is registered, and `"no_providers"` (`503`) only when none are — it does not track whether any target is actually routable (that's what `/readyz` is for) and it performs no live upstream probe. `circuit` reports `"closed"` both for a genuinely closed breaker and for a provider with no breaker configured at all — use `gateway_circuit_breaker_state` to tell those apart, since a series exists only where a breaker is configured. The dashboard's Providers page polls this endpoint. + +For orchestrator wiring (Kubernetes probes, load-balancer health checks) and the request-log stats behind the dashboard's Analytics page, see [Monitoring](/operations/monitoring) and [Request logging](/operations/request-logging). + +## Related -For monitoring in production, see [Monitoring](/operations/monitoring). +- [The embedded dashboard](/guides/dashboard) — the built-in Tracing and Analytics pages, and `make up-fullstack` for a local Jaeger + Prometheus + Grafana stack +- [Authentication](/guides/auth) — scopes, `MASTER_KEY`, and issuing a `read_only` scrape key +- [Monitoring](/operations/monitoring) — orchestrator probe wiring for `/livez` and `/readyz` +- [Request logging](/operations/request-logging) — the persisted `duration_ms` / `ttft_ms` / `cost_usd` request-log columns and `GET /admin/logs/stats` +- [MCP integration](/guides/mcp) — `mcp_servers[].required` and readiness gating diff --git a/docs/guides/oss-vs-ferrocloud.mdx b/docs/guides/oss-vs-ferrocloud.mdx index d875793..35e51a7 100644 --- a/docs/guides/oss-vs-ferrocloud.mdx +++ b/docs/guides/oss-vs-ferrocloud.mdx @@ -1,6 +1,6 @@ --- title: OSS vs Ferro Labs Managed -description: "Side-by-side comparison of Ferro Labs AI Gateway open-source and Ferro Labs Managed managed service — features, plugins, plans, pricing, and when to upgrade from self-hosted to managed." +description: "Compare Ferro Labs AI Gateway open source with Ferro Labs Managed — features, plugins, plans, and when to move from self-hosted to managed." keywords: [Ferro Gateway OSS vs Cloud, Ferro Labs Managed comparison, AI gateway self-hosted vs managed, open-source LLM proxy comparison, Ferro Labs Managed] --- diff --git a/docs/guides/plugins.mdx b/docs/guides/plugins.mdx deleted file mode 100644 index f693080..0000000 --- a/docs/guides/plugins.mdx +++ /dev/null @@ -1,265 +0,0 @@ ---- -title: Plugins -description: The 6 open-source built-in plugins in the Ferro Labs AI Gateway — word filtering, token limits, response caching, request logging, rate limiting, and spend budgets. Advanced security plugins (PII redaction, prompt injection shield, secret scanning) are available in Ferro Labs Managed. -keywords: [AI gateway plugins, LLM plugins OSS, response caching LLM, request logging AI, rate limiting AI gateway, spend budget AI, word filter LLM] ---- - -Plugins extend the request pipeline at three lifecycle stages: - -- `before_request` — runs before the request is forwarded to the provider -- `after_request` — runs after the provider response is received -- `on_error` — runs when the provider returns an error - -Each plugin entry in `config.yaml` has `name`, `type`, `stage`, `enabled`, and an optional `config` map. Disabled plugins (`enabled: false`) are ignored at runtime. - -| Plugin | Type | Stage | Open-source | -|---|---|---|---| -| `word-filter` | guardrail | before_request | ✅ | -| `max-token` | guardrail | before_request | ✅ | -| `response-cache` | transform | before_request | ✅ | -| `request-logger` | logging | before_request | ✅ | -| `rate-limit` | ratelimit | before_request | ✅ | -| `budget` | guardrail | before_request + after_request | ✅ | -| `pii-redact` | guardrail | before_request | [Ferro Labs Managed only](/guides/oss-vs-ferrocloud) | -| `secret-scan` | guardrail | before_request | [Ferro Labs Managed only](/guides/oss-vs-ferrocloud) | -| `prompt-shield` | guardrail | before_request | [Ferro Labs Managed only](/guides/oss-vs-ferrocloud) | -| `schema-guard` | guardrail | after_request | [Ferro Labs Managed only](/guides/oss-vs-ferrocloud) | -| `regex-guard` | guardrail | before_request | [Ferro Labs Managed only](/guides/oss-vs-ferrocloud) | - -## Guardrail plugins - -### word-filter - -Blocks requests whose messages contain any of the configured words or phrases. Case sensitivity is optional. - -```yaml -- name: word-filter - type: guardrail - stage: before_request - enabled: true - config: - blocked_words: ["confidential", "password", "secret"] - case_sensitive: false -``` - -### max-token - -Enforces limits on token count, message count, and raw input length before the request reaches the provider. - -```yaml -- name: max-token - type: guardrail - stage: before_request - enabled: true - config: - max_tokens: 4096 # maximum output tokens to request - max_messages: 50 # maximum messages in the conversation - max_input_length: 20000 # maximum raw characters in user input -``` - -### pii-redact - -:::note Ferro Labs Managed only -This plugin is available in [Ferro Labs Managed](https://ferrolabs.ai) managed deployments. It is not included in the open-source gateway. -::: - -Detects Personally Identifiable Information (PII) in the request and either redacts it or blocks the request entirely. - -```yaml -- name: pii-redact - type: guardrail - stage: before_request - enabled: true - config: - action: redact # redact | block - redact_mode: replace_type # replace detected entity with its type label - apply_to: input # input | output | both - entities: [] # empty = detect all entity types -``` - -With `action: redact`, detected PII is replaced in-place before the request is forwarded. With `action: block`, the entire request is rejected with a `400` error. - -### secret-scan - -:::note Ferro Labs Managed only -This plugin is available in [Ferro Labs Managed](https://ferrolabs.ai) managed deployments. It is not included in the open-source gateway. -::: - -Scans request content for leaked credentials, API keys, and secrets using pattern matching and (optionally) entropy analysis. - -```yaml -- name: secret-scan - type: guardrail - stage: before_request - enabled: true - config: - action: block # block | warn - entropy_check: true # also flag high-entropy strings -``` - -### prompt-shield - -:::note Ferro Labs Managed only -This plugin is available in [Ferro Labs Managed](https://ferrolabs.ai) managed deployments. It is not included in the open-source gateway. -::: - -Scores user messages for prompt injection attempts and blocks requests that exceed a configurable confidence threshold. - -```yaml -- name: prompt-shield - type: guardrail - stage: before_request - enabled: true - config: - action: block - threshold: 0.90 # 0.0–1.0; higher = stricter - apply_to: user_messages -``` - -### schema-guard - -:::note Ferro Labs Managed only -This plugin is available in [Ferro Labs Managed](https://ferrolabs.ai) managed deployments. It is not included in the open-source gateway. -::: - -Validates the model's JSON output against a JSON Schema. Runs `after_request`. Optionally extracts JSON from a text response before validating. - -```yaml -- name: schema-guard - type: guardrail - stage: after_request - enabled: true - config: - apply_to: output - action: block - extract_json: true # attempt to parse JSON from a markdown code block - schema: - type: object - required: [name, confidence] - properties: - name: - type: string - confidence: - type: number - minimum: 0 - maximum: 1 -``` - -### regex-guard - -:::note Ferro Labs Managed only -This plugin is available in [Ferro Labs Managed](https://ferrolabs.ai) managed deployments. It is not included in the open-source gateway. -::: - -Blocks or warns on requests matching one or more regular expressions. Useful for custom business rules not covered by other guardrails. - -```yaml -- name: regex-guard - type: guardrail - stage: before_request - enabled: true - config: - action: block # block | warn - rules: - - pattern: "(?i)(ssn|social security)\\s*:?\\s*\\d{3}-\\d{2}-\\d{4}" - message: "SSN pattern detected" - - pattern: "(?i)jailbreak|ignore previous instructions" - message: "Potential jailbreak attempt" -``` - -## Transform plugins - -### response-cache - -Caches exact-match responses in memory. Identical requests (same model + messages) served from cache skip the provider entirely. - -```yaml -- name: response-cache - type: transform - stage: before_request - enabled: true - config: - max_age: 300 # seconds before a cache entry expires - max_entries: 1000 # maximum number of cached responses -``` - -## Logging plugins - -### request-logger - -Emits structured per-request logs. Optionally persists request and response data to SQLite or Postgres for later querying via the admin API. - -```yaml -- name: request-logger - type: logging - stage: before_request - enabled: true - config: - level: info - persist: true - backend: sqlite # sqlite | postgres - dsn: ferrogw-requests.db # SQLite path or Postgres DSN -``` - -When `persist: true`, requests are queryable at `GET /admin/logs`. See [Request logging](/operations/request-logging). - -## Rate limit plugins - -### rate-limit - -Token-bucket rate limiting applied per request. Rejects requests with `429 Too Many Requests` when the bucket is empty. - -```yaml -- name: rate-limit - type: ratelimit - stage: before_request - enabled: true - config: - requests_per_second: 50 - burst: 100 -``` - -For IP-level rate limiting (HTTP middleware layer), see [Rate limiting](/guides/rate-limiting). - -## Budget plugins - -### budget - -Tracks cumulative USD spend per API key using an in-memory token-cost model. Must be registered at **both** `before_request` (to check the limit) and `after_request` (to record the cost). The two instances share state via `store_id`. - -Requests without an `api_key` in request metadata are not subject to budget enforcement. - -:::info In-memory only -Spend data is in-memory and resets on gateway restart. Use this for session-scoped soft limits and development quotas. Durable billing enforcement is available in Ferro Labs Managed. -::: - -```yaml -plugins: - # Check limit before forwarding - - name: budget - type: guardrail - stage: before_request - enabled: true - config: - store_id: "default" # shared between before/after instances - spend_limit_usd: 50.0 # max cumulative spend per API key (USD) - input_per_m_tokens: 3.0 # cost per 1M prompt tokens - output_per_m_tokens: 15.0 # cost per 1M completion tokens - max_keys: 10000 # max tracked API keys before eviction - - # Record cost after response - - name: budget - type: guardrail - stage: after_request - enabled: true - config: - store_id: "default" # must match the before_request instance - input_per_m_tokens: 3.0 - output_per_m_tokens: 15.0 -``` - -When the accumulated spend for an API key reaches `spend_limit_usd`, subsequent requests are rejected with `HTTP 429`. - -## Plugin execution order - -Plugins are executed in the order they appear in `config.yaml`. Within a stage, if any plugin sets `reject: true`, execution stops and an error is returned to the client. Setting `skip: true` short-circuits the **entire remaining stage loop** — the current plugin finishes, then all subsequent plugins in that stage are skipped (it does not merely bypass the current plugin). This is how the `response-cache` plugin short-circuits the rest of the `before_request` stage and the provider call on a cache hit. diff --git a/docs/guides/prompt-templates.mdx b/docs/guides/prompt-templates.mdx index d7fe7ae..382eab8 100644 --- a/docs/guides/prompt-templates.mdx +++ b/docs/guides/prompt-templates.mdx @@ -1,91 +1,9 @@ --- title: Prompt templates -description: Use the Ferro Labs AI Gateway prompt template API to define reusable system prompts and inject variables at request time — reduce repetition and enforce prompt consistency across teams. -keywords: [prompt templates AI gateway, LLM prompt management, reusable prompts, prompt variables, system prompt injection] +description: "Server-side prompt templates — reusable, versioned system prompts referenced by ID — are a planned Ferro Labs Managed feature, unavailable in the OSS gateway." +keywords: [prompt templates, Ferro Labs Managed, reusable prompts, system prompt injection, transform plugin] --- -:::note Ferro Labs Managed only -Server-side prompt templates are a **Ferro Labs Managed** feature. They are not available in the open-source gateway. The configuration and API below apply to the Managed offering — see [Ferro Labs Managed](/ferrocloud/overview). +:::info Ferro Labs Managed +Server-side prompt templates — named, versioned system prompts you reference by ID at request time instead of resending in every call — are a planned **Ferro Labs Managed** capability and have not shipped yet. They do not exist in the open-source gateway: `prompt_templates` is not a recognized config key, and the OSS gateway's strict config decoding rejects any unknown key, so a `config.yaml` declaring one fails `ferrogw validate`. See [Ferro Labs Managed](/ferrocloud/overview) for what's planned. The closest mechanism available in the OSS gateway today is a [transform plugin](/plugins), which can rewrite a request — including prepending a system message — before it reaches the provider. ::: - -Prompt templates let you define named, versioned prompts server-side and reference them by ID at request time. The gateway injects the rendered template into the request before forwarding it to the provider — clients never need to re-send long system prompts. - -## Defining a template - -Add a `prompt_templates` block to `config.yaml`: - -```yaml -prompt_templates: - - id: customer-support - description: "Tier-1 customer support agent" - template: | - You are a helpful customer support agent for {{company_name}}. - Always respond in {{language}}. - Escalation email: {{escalation_email}} - - - id: code-reviewer - description: "Code review assistant" - template: | - You are an expert {{language}} code reviewer. - Focus on: correctness, security, and readability. - Be concise. Use inline code fences for suggestions. -``` - -Variable placeholders use double-brace syntax: `{{variable_name}}`. - -## Using a template in a request - -Pass `template_id` and `template_vars` in the request body alongside or instead of a system message: - -```python -from openai import OpenAI - -client = OpenAI( - api_key="sk-your-key", - base_url="http://localhost:8080", -) - -response = client.chat.completions.create( - model="gpt-4o-mini", - messages=[{"role": "user", "content": "My order #12345 hasn't arrived."}], - extra_body={ - "template_id": "customer-support", - "template_vars": { - "company_name": "Acme Corp", - "language": "English", - "escalation_email": "support-l2@acme.example", - }, - }, -) -print(response.choices[0].message.content) -``` - -The gateway renders the template, prepends it as the system message, and forwards the enriched request to the provider. - -## Mixing templates with system messages - -If the request already has a system message, the rendered template is prepended before the existing system message. The existing message is preserved. - -```python -# Rendered template becomes system[0]; your message becomes system[1] -messages = [ - {"role": "system", "content": "Additional context for this request."}, - {"role": "user", "content": "Review this Go function for security issues."}, -] -``` - -## Listing templates via the admin API - -```bash -curl -H "Authorization: Bearer $ADMIN_API_KEY" \ - http://localhost:8080/admin/templates -``` - -Returns an array of all configured template IDs, descriptions, and variable names (extracted automatically from the template). - -## Best practices - -- **Keep templates in version control.** Store them in `config.yaml` and track changes alongside your gateway configuration. -- **Use descriptive IDs.** `customer-support-en` is clearer than `tmpl-001`. -- **Validate variables.** The gateway returns `400 Bad Request` if a required variable is missing from `template_vars`. -- **Avoid secrets in templates.** Don't hardcode API keys or passwords — inject them from environment variables instead. diff --git a/docs/guides/provider-capabilities.mdx b/docs/guides/provider-capabilities.mdx deleted file mode 100644 index e3cd7f2..0000000 --- a/docs/guides/provider-capabilities.mdx +++ /dev/null @@ -1,77 +0,0 @@ ---- -title: Provider Capability Matrix -description: Provider capability matrix for the Ferro Labs AI Gateway — chat, streaming, embeddings, image generation, model discovery, and proxy support across 30 providers. -keywords: [provider capabilities, capability matrix, embeddings, image generation, model discovery, streaming, passthrough proxy, AI gateway providers] ---- - -The gateway exposes a single OpenAI-compatible surface, but not every provider implements every capability. This page is a **provider × capability matrix** generated from the gateway's built-in provider registry (`providers/providers_list.go`), so it always reflects exactly what each provider actually advertises. - -## What the capabilities mean - -- **Chat** — chat completions via `POST /v1/chat/completions`. Every provider supports this. -- **Stream** — incremental token streaming with `"stream": true` (server-sent events). -- **Embed** — vector embeddings for retrieval, search, and clustering. -- **Image** — image generation from a text prompt. -- **Discovery** — live model discovery, so `GET /v1/models` reflects the provider's current catalog rather than only the static built-in catalog. -- **Proxy** — transparent passthrough of unhandled `/v1/*` endpoints straight to the provider, preserving provider-native request and response shapes. - -Each row corresponds to a provider's virtual key. A ✅ means the provider declares that capability in the registry; a — means it does not. - -## The matrix - -| Provider | Virtual Key | Chat | Stream | Embed | Image | Discovery | Proxy | -|---|---|:--:|:--:|:--:|:--:|:--:|:--:| -| **AI21** | `ai21` | ✅ | ✅ | — | — | — | ✅ | -| **Anthropic** | `anthropic` | ✅ | ✅ | — | — | — | ✅ | -| **Azure Foundry** | `azure-foundry` | ✅ | ✅ | — | — | — | ✅ | -| **Azure OpenAI** | `azure-openai` | ✅ | ✅ | — | — | — | ✅ | -| **AWS Bedrock** | `bedrock` | ✅ | ✅ | ✅ | — | — | ✅ | -| **Cerebras** | `cerebras` | ✅ | ✅ | — | — | ✅ | ✅ | -| **Cloudflare Workers AI** | `cloudflare` | ✅ | ✅ | ✅ | — | — | ✅ | -| **Cohere** | `cohere` | ✅ | ✅ | ✅ | — | — | ✅ | -| **Databricks** | `databricks` | ✅ | ✅ | ✅ | — | — | ✅ | -| **DeepInfra** | `deepinfra` | ✅ | ✅ | — | — | — | ✅ | -| **DeepSeek** | `deepseek` | ✅ | ✅ | — | — | — | ✅ | -| **Fireworks AI** | `fireworks` | ✅ | ✅ | ✅ | — | ✅ | ✅ | -| **Google Gemini** | `gemini` | ✅ | ✅ | ✅ | — | — | ✅ | -| **Groq** | `groq` | ✅ | ✅ | — | — | — | ✅ | -| **Hugging Face** | `hugging-face` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| **Mistral** | `mistral` | ✅ | ✅ | ✅ | — | — | ✅ | -| **Moonshot AI** | `moonshot` | ✅ | ✅ | — | — | — | ✅ | -| **Novita AI** | `novita` | ✅ | ✅ | ✅ | — | ✅ | ✅ | -| **NVIDIA NIM** | `nvidia-nim` | ✅ | ✅ | — | — | — | ✅ | -| **Ollama** | `ollama` | ✅ | ✅ | — | — | — | ✅ | -| **Ollama Cloud** | `ollama-cloud` | ✅ | ✅ | — | — | ✅ | — | -| **OpenAI** | `openai` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| **OpenRouter** | `openrouter` | ✅ | ✅ | — | — | ✅ | ✅ | -| **Perplexity** | `perplexity` | ✅ | ✅ | — | — | ✅ | ✅ | -| **Qwen (Alibaba)** | `qwen` | ✅ | ✅ | — | — | — | ✅ | -| **Replicate** | `replicate` | ✅ | — | — | ✅ | — | ✅ | -| **SambaNova** | `sambanova` | ✅ | ✅ | — | — | — | ✅ | -| **Together AI** | `together` | ✅ | ✅ | ✅ | — | — | ✅ | -| **Vertex AI** | `vertex-ai` | ✅ | ✅ | ✅ | — | — | ✅ | -| **xAI (Grok)** | `xai` | ✅ | ✅ | — | — | ✅ | ✅ | - -## Capability totals - -Across the 30 built-in providers: - -- **Chat** — 30 providers -- **Stream** — 29 providers (every provider except Replicate) -- **Embed** — 11 providers -- **Image** — 3 providers (Hugging Face, OpenAI, Replicate) -- **Discovery** — 9 providers -- **Proxy** — 29 providers (every provider except Ollama Cloud) - -## Notes - -- **Chat is universal.** All 30 providers implement chat completions, so the gateway can route any request to any of them under your chosen strategy. -- **Streaming is near-universal.** Only Replicate does not stream, because its text and image models run as polled predictions rather than token streams. -- **Discovery vs. the static catalog.** Providers without Discovery still return models — they fall back to the built-in model catalog used for cost estimation and `GET /v1/models` — but they will not reflect newly released provider models until the catalog is updated. -- **Proxy reach.** Almost every provider allows passthrough of provider-native endpoints; Ollama Cloud is the exception and is reached through the gateway's normalized chat and discovery paths only. - -## Next steps - -- [Providers](/guides/providers) — the full list of providers and required environment variables. -- [Provider configuration](/guides/providers-config) — environment variables and setup details for each provider. -- [Routing policies](/guides/routing-policies) — how to route and fail over across providers. diff --git a/docs/guides/providers-config.mdx b/docs/guides/providers-config.mdx deleted file mode 100644 index 707cb76..0000000 --- a/docs/guides/providers-config.mdx +++ /dev/null @@ -1,197 +0,0 @@ ---- -title: Provider configuration -description: Environment variables and credentials for all 30 AI providers in the Ferro Labs AI Gateway — API keys, base URLs, and model lists for OpenAI, Anthropic, Gemini, Ollama, Cerebras, NVIDIA NIM, Cloudflare, Databricks, and more. -keywords: [AI provider configuration, OpenAI API key setup, Anthropic setup, LLM provider credentials, gateway environment variables, provider API keys, Cerebras, NVIDIA NIM, Cloudflare Workers AI, Databricks, OpenRouter] ---- - -Providers are automatically registered when their required environment variables are present at startup. Set at least one provider key before starting the gateway. - -## Simple providers (API key only) - -These providers require a single API key: - -```bash -export OPENAI_API_KEY=sk-... -export ANTHROPIC_API_KEY=sk-ant-... -export GEMINI_API_KEY=... -export MISTRAL_API_KEY=... -export GROQ_API_KEY=gsk-... -export COHERE_API_KEY=... -export DEEPSEEK_API_KEY=... -export TOGETHER_API_KEY=... -export PERPLEXITY_API_KEY=pplx-... -export FIREWORKS_API_KEY=fw-... -export AI21_API_KEY=... -export XAI_API_KEY=xai-... -export HUGGING_FACE_API_KEY=hf-... -export CEREBRAS_API_KEY=... -export NVIDIA_NIM_API_KEY=... -export NOVITA_API_KEY=... -export QWEN_API_KEY=... -export MOONSHOT_API_KEY=... -export SAMBANOVA_API_KEY=... -export DEEPINFRA_API_KEY=... -export OPENROUTER_API_KEY=... -``` - -Most providers also accept an optional `_BASE_URL` override (for example `OPENAI_BASE_URL`) to point at a self-hosted or proxy endpoint. The exception is Hugging Face, whose override variable is `HUGGING_FACE_ENDPOINT` rather than a `_BASE_URL` variant. - -## Azure OpenAI - -Requires endpoint, deployment name, and API version in addition to the key: - -```bash -export AZURE_OPENAI_API_KEY=... -export AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com -export AZURE_OPENAI_DEPLOYMENT=gpt-4o -export AZURE_OPENAI_API_VERSION=2024-10-21 -``` - -## Azure AI Foundry - -```bash -export AZURE_FOUNDRY_API_KEY=... -export AZURE_FOUNDRY_ENDPOINT=https://your-project.services.ai.azure.com -``` - -## Ollama (local / self-hosted) - -Ollama does not require an API key. The host defaults to `http://localhost:11434`. - -```bash -export OLLAMA_HOST=http://localhost:11434 -export OLLAMA_MODELS=llama3.2,llama3.1,mistral -``` - -`OLLAMA_MODELS` is a comma-separated list of model tags to expose at `/v1/models`. - -## Ollama Cloud (hosted) - -Ollama Cloud requires an API key. The base URL and exposed models are optional overrides. - -```bash -export OLLAMA_API_KEY=... -export OLLAMA_CLOUD_BASE_URL=https://ollama.com -export OLLAMA_CLOUD_MODELS=gpt-oss:20b -``` - -`OLLAMA_CLOUD_MODELS` is a comma-separated list of model tags to expose at `/v1/models`. - -## Replicate - -```bash -export REPLICATE_API_TOKEN=r8_... -export REPLICATE_TEXT_MODELS=meta/llama-3-8b-instruct,mistralai/mistral-7b-instruct-v0.2 -export REPLICATE_IMAGE_MODELS=stability-ai/sdxl -``` - -`REPLICATE_TEXT_MODELS` and `REPLICATE_IMAGE_MODELS` are comma-separated lists of Replicate model IDs to register. - -## AWS Bedrock - -Bedrock uses the standard AWS credential chain. Configure either `AWS_REGION` alone (for IAM roles / instance profiles) or full static credentials: - -```bash -# Option 1 — IAM role or instance profile -export AWS_REGION=us-east-1 - -# Option 2 — static credentials -export AWS_REGION=us-east-1 -export AWS_ACCESS_KEY_ID=AKIA... -export AWS_SECRET_ACCESS_KEY=... -``` - -## Google Vertex AI - -Vertex AI requires the project ID and region, plus **one** of an API key or a service account JSON for authentication: - -```bash -export VERTEX_AI_PROJECT_ID=my-gcp-project -export VERTEX_AI_REGION=us-central1 - -# Option 1 — API key -export VERTEX_AI_API_KEY=... - -# Option 2 — service account JSON -export VERTEX_AI_SERVICE_ACCOUNT_JSON='{"type":"service_account",...}' -``` - -## Cloudflare Workers AI - -Requires both your Cloudflare account ID and an API key with Workers AI permissions: - -```bash -export CLOUDFLARE_ACCOUNT_ID=... -export CLOUDFLARE_API_KEY=... -``` - -## Databricks - -Use the workspace URL (without trailing slash) and a personal access token or service principal token: - -```bash -export DATABRICKS_HOST=https://your-workspace.azuredatabricks.net -export DATABRICKS_TOKEN=dapi... -``` - -## Cerebras - -```bash -export CEREBRAS_API_KEY=... -``` - -## NVIDIA NIM - -```bash -export NVIDIA_NIM_API_KEY=... -``` - -## Novita AI - -```bash -export NOVITA_API_KEY=... -``` - -## Qwen (Alibaba Cloud) - -```bash -export QWEN_API_KEY=... -``` - -## Moonshot AI - -```bash -export MOONSHOT_API_KEY=... -``` - -## SambaNova - -```bash -export SAMBANOVA_API_KEY=... -``` - -## DeepInfra - -```bash -export DEEPINFRA_API_KEY=... -``` - -## OpenRouter - -```bash -export OPENROUTER_API_KEY=... -``` - -## Validate configured providers - -After starting the gateway, confirm which providers are active: - -```bash -# List all registered models (grouped by provider) -curl http://localhost:8080/v1/models - -# Deep health check with per-provider status -curl http://localhost:8080/health -``` - -The `/health` endpoint returns `200 OK` if at least one provider is reachable and includes per-provider latency in the JSON body. diff --git a/docs/guides/providers.mdx b/docs/guides/providers.mdx deleted file mode 100644 index afe946c..0000000 --- a/docs/guides/providers.mdx +++ /dev/null @@ -1,75 +0,0 @@ ---- -title: Providers -description: All 30 AI providers supported by the Ferro Labs AI Gateway — OpenAI, Anthropic, Gemini, Mistral, Groq, DeepSeek, Ollama, AWS Bedrock, Azure OpenAI, Cerebras, NVIDIA NIM, Cloudflare, Databricks, and more with virtual key routing. -keywords: [AI providers, OpenAI proxy, Anthropic gateway, Gemini proxy, Mistral gateway, Groq proxy, multi-LLM gateway, Ollama proxy, Bedrock gateway, DeepSeek proxy, Cerebras, NVIDIA NIM, Cloudflare Workers AI, Databricks, OpenRouter] ---- - -The gateway supports 30 AI providers. A provider becomes active when its required environment variables are set — no code changes or rebuilds needed. - -## All providers - -| Provider | Virtual Key | Required Environment Variables | -|---|---|---| -| **OpenAI** | `openai` | `OPENAI_API_KEY` | -| **Anthropic** | `anthropic` | `ANTHROPIC_API_KEY` | -| **Google Gemini** | `gemini` | `GEMINI_API_KEY` | -| **Mistral** | `mistral` | `MISTRAL_API_KEY` | -| **Groq** | `groq` | `GROQ_API_KEY` | -| **Cohere** | `cohere` | `COHERE_API_KEY` | -| **DeepSeek** | `deepseek` | `DEEPSEEK_API_KEY` | -| **Together AI** | `together` | `TOGETHER_API_KEY` | -| **Perplexity** | `perplexity` | `PERPLEXITY_API_KEY` | -| **Fireworks AI** | `fireworks` | `FIREWORKS_API_KEY` | -| **AI21** | `ai21` | `AI21_API_KEY` | -| **xAI (Grok)** | `xai` | `XAI_API_KEY` | -| **Azure OpenAI** | `azure-openai` | `AZURE_OPENAI_API_KEY` + endpoint + deployment | -| **Azure Foundry** | `azure-foundry` | `AZURE_FOUNDRY_API_KEY` + `AZURE_FOUNDRY_ENDPOINT` | -| **Ollama** | `ollama` | `OLLAMA_HOST` (no API key required) | -| **Ollama Cloud** | `ollama-cloud` | `OLLAMA_API_KEY` | -| **AWS Bedrock** | `bedrock` | `AWS_REGION` or `AWS_ACCESS_KEY_ID` | -| **Replicate** | `replicate` | `REPLICATE_API_TOKEN` | -| **Vertex AI** | `vertex-ai` | `VERTEX_AI_PROJECT_ID` + `VERTEX_AI_REGION` + (`VERTEX_AI_API_KEY` or `VERTEX_AI_SERVICE_ACCOUNT_JSON`) | -| **Hugging Face** | `hugging-face` | `HUGGING_FACE_API_KEY` | -| **Cerebras** | `cerebras` | `CEREBRAS_API_KEY` | -| **NVIDIA NIM** | `nvidia-nim` | `NVIDIA_NIM_API_KEY` | -| **Cloudflare Workers AI** | `cloudflare` | `CLOUDFLARE_ACCOUNT_ID` + `CLOUDFLARE_API_KEY` | -| **Databricks** | `databricks` | `DATABRICKS_HOST` + `DATABRICKS_TOKEN` | -| **Novita AI** | `novita` | `NOVITA_API_KEY` | -| **Qwen (Alibaba)** | `qwen` | `QWEN_API_KEY` | -| **Moonshot AI** | `moonshot` | `MOONSHOT_API_KEY` | -| **SambaNova** | `sambanova` | `SAMBANOVA_API_KEY` | -| **DeepInfra** | `deepinfra` | `DEEPINFRA_API_KEY` | -| **OpenRouter** | `openrouter` | `OPENROUTER_API_KEY` | - -## Provider capabilities - -All providers support chat completions and streaming. Capability support varies by provider: - -- **Embeddings** — AWS Bedrock, Cloudflare Workers AI, Cohere, Databricks, Fireworks AI, Google Gemini, Hugging Face, Mistral, Novita AI, OpenAI, Together AI -- **Image generation** — OpenAI (DALL·E), Hugging Face, Replicate -- **Local / self-hosted** — Ollama -- **Managed cloud inference** — AWS Bedrock, Vertex AI, Azure Foundry, Databricks -- **High-speed inference** — Cerebras, SambaNova, Groq -- **Model aggregators** — OpenRouter, Novita AI (access hundreds of models via one key) - -## Provider selection at runtime - -The gateway selects a provider using the configured routing strategy. You can also force a specific provider for a single request using the `X-Provider` request header: - -```bash -curl http://localhost:8080/v1/chat/completions \ - -H "X-Provider: anthropic" \ - -H "Content-Type: application/json" \ - -d '{"model": "claude-3-5-sonnet-20241022", "messages": [{"role": "user", "content": "Hi"}]}' -``` - -If `X-Provider` is set, the routing strategy is bypassed for that request. - -## Model catalog - -The gateway ships with a built-in catalog of 2,500+ model entries used for cost estimation and the `/v1/models` response. Run `GET /v1/models` to see all available models given your configured providers. - -## Next steps - -- [Provider configuration](/guides/providers-config) — environment variables for each provider -- [Routing policies](/guides/routing-policies) — how to route across providers diff --git a/docs/guides/rate-limiting.mdx b/docs/guides/rate-limiting.mdx index d004fdc..6ddd714 100644 --- a/docs/guides/rate-limiting.mdx +++ b/docs/guides/rate-limiting.mdx @@ -1,28 +1,40 @@ --- title: Rate limiting -description: Control AI request volume in the Ferro Labs AI Gateway with per-IP HTTP rate limiting, a global token-bucket plugin, and optional per-API-key and per-user request budgets to protect your LLM spend. -keywords: [AI rate limiting, LLM rate limit, per-IP rate limiting, per-key rate limiting, per-user rate limiting, AI gateway throttling, token rate limiting, request budget control] +description: Configure per-IP HTTP throttling and the rate-limit plugin in the Ferro Labs AI Gateway — trusted-proxy IP resolution, Retry-After, and per-key/user limits. +keywords: [AI rate limiting, LLM rate limit, per-IP rate limiting, per-key rate limiting, per-user rate limiting, AI gateway throttling, token rate limiting, provider concurrency limit] --- The gateway rate-limits in two independent places: -1. **Per-IP HTTP middleware** — runs at the edge, before routing, configured by environment variables. -2. **The `rate-limit` plugin** — runs at the `before_request` stage, configured in your gateway config. It layers a global limiter with optional per-API-key and per-user limiters. +1. **Per-IP HTTP middleware** — runs at the edge, before routing, configured by environment variables. **On by default.** +2. **The `rate-limit` plugin** — runs at the `before_request` stage, configured in your gateway config. It layers a global limiter with optional per-API-key and per-user limiters. Off by default. -These two layers are separate systems. The middleware keys on client IP; the plugin keys on global traffic, API key, and user ID. You can run either, both, or neither. +These two layers are separate systems. The middleware keys on client IP; the plugin keys on global traffic, API key, and user ID. You can run either, both, or neither. A third rejection — `provider_saturated` — is rate-shaped but comes from a different setting entirely; see [Other 429s and the 402 budget response](#other-429s-and-the-402-budget-response) below. ## Per-IP HTTP middleware -This layer is wired into the HTTP router and keyed on the client IP. It is enabled only when `RATE_LIMIT_RPS` is set to a positive number: +This layer is wired into the HTTP router and keyed on the client IP. **It is enabled by default** at 20 requests/second with a burst of 40 — no configuration is required to get it: ```bash -export RATE_LIMIT_RPS=20 # required to enable; per-IP requests/second -export RATE_LIMIT_BURST=40 # optional burst capacity (defaults to RATE_LIMIT_RPS when unset) +export RATE_LIMIT_RPS=20 # default 20; set to 0 to disable the middleware entirely +export RATE_LIMIT_BURST=40 # default 40 ``` -The client IP is taken from the connection's remote address, or from the first entry of the `X-Forwarded-For` header when present (set this only behind a trusted proxy). +Setting `RATE_LIMIT_RPS` alone always resets the burst back to the default of 40 too — set `RATE_LIMIT_BURST` explicitly alongside it for a custom rate/burst combination. The store tracks up to 100,000 distinct IPs; invalid values for either variable are ignored with a startup warning and fall back to the default. -When an IP exceeds its bucket, the request is rejected with HTTP `429 Too Many Requests` and an OpenAI-style JSON error body: +:::note +`RATE_LIMIT_RPS=0` is a legitimate way to turn the middleware off — for example when rate limiting is enforced at an ingress or upstream API gateway instead. Under `GATEWAY_ENV=production` this only produces a startup **warning**, not a refusal to boot. +::: + +The client IP is **not** read as the leftmost entry of `X-Forwarded-For`. It is resolved from the trusted-proxy chain: `X-Forwarded-For`/`X-Real-IP` is honored only when the direct TCP peer is inside a trusted-proxy CIDR (`TRUSTED_PROXIES`, default `127.0.0.0/8,::1/128` — loopback only), and even then walks the `X-Forwarded-For` chain from the **right**, taking the first hop that isn't itself a trusted proxy. Proxies append to the chain, so the rightmost untrusted entry is the one an actual trusted hop observed; the leftmost entry is whatever the original caller chose to send and is never trusted. Reading the chain the other way — from the left — was a spoofing bug fixed in v1.4.0: a caller could set its own `X-Forwarded-For` header to mint a fresh IP (and therefore a fresh bucket) on every request. + +Deploy behind a reverse proxy or load balancer outside the default loopback range and set `TRUSTED_PROXIES` to its real CIDR — otherwise every request resolves to the proxy's own IP and the whole layer collapses into one shared bucket for all clients. + +```bash +export TRUSTED_PROXIES=10.0.0.0/8 # CIDRs of your reverse proxy / ingress / sidecar +``` + +When an IP exceeds its bucket, the request is rejected with HTTP `429 Too Many Requests`, a `Retry-After: 1` header, and an OpenAI-style JSON error body: ```json { @@ -34,11 +46,11 @@ When an IP exceeds its bucket, the request is rejected with HTTP `429 Too Many R } ``` -:::note -The `429` response does **not** include a `Retry-After` header. Clients should back off using their own retry/jitter policy rather than relying on a server-provided delay. -::: +Every 429 the gateway decides on its own — this middleware, the plugin below, and a saturated provider target — carries the same `Retry-After: 1`. An upstream provider's own `Retry-After` hint, when one is present on a proxied error, is propagated instead of the constant. -If `RATE_LIMIT_RPS` is unset (or non-positive), the per-IP middleware is disabled entirely. +:::tip +`POST /admin/session` — the dashboard sign-in endpoint — carries its own independent limiter (10 requests/minute, burst 20, keyed on IP) that is unaffected by `RATE_LIMIT_RPS=0`. It is the only unauthenticated write path on the gateway, so it cannot be switched off while tuning inference throughput. +::: ## The `rate-limit` plugin @@ -73,12 +85,18 @@ Set `enabled: true` to turn it on. The plugin uses in-memory token buckets, so l | `user_rpm` | unset (off) | Per user, requests/minute | request `user` field (`Request.User`) | - **`requests_per_second`** — the global rate, always active. Every request consumes one token from this bucket regardless of key or user. -- **`burst`** — global burst capacity. When unset (or `≤ 0`) it defaults to `requests_per_second`, meaning no extra headroom above the steady rate. -- **`key_rpm`** — optional. Caps requests per minute for each distinct API key. Internally the bucket refills at `key_rpm / 60` tokens per second with a burst of `key_rpm`, so an idle key can spend up to a full minute's worth of requests at once. Must be `> 0` if set. -- **`user_rpm`** — optional. Caps requests per minute for each distinct user ID, using the same refill/burst semantics as `key_rpm`. Must be `> 0` if set. +- **`burst`** — global burst capacity. When unset it defaults to `requests_per_second`, meaning no extra headroom above the steady rate. +- **`key_rpm`** — optional. Caps requests per minute for each distinct API key. Internally the bucket refills at `key_rpm / 60` tokens per second with a burst of `key_rpm`, so an idle key can spend up to a full minute's worth of requests at once. +- **`user_rpm`** — optional. Caps requests per minute for each distinct user ID, using the same refill/burst semantics as `key_rpm`. The per-key and per-user stores track up to 100,000 distinct keys each, evicting the least recently used entry beyond that cap to bound memory. +:::warning +Every field you set — `requests_per_second`, `burst`, `key_rpm`, `user_rpm` — must be a **positive** number. `0` (or a negative value, `NaN`, or `Inf`) is rejected at load by both the gateway and `ferrogw validate`, because each field is a rate: a rate of zero blackholes every request forever rather than acting as "off," and a gateway that started up and reported healthy while silently rejecting all traffic is the worst shape a config mistake can take. Turn the plugin off with `enabled: false` instead. + +This is the deliberate opposite of `RATE_LIMIT_RPS=0` for the per-IP middleware above: there the environment variable *is* the whole switch, so `0` means "no limiting." Here each field is one setting inside a plugin that already has its own switch. +::: + ### Evaluation order Checks run in a fixed order, and the request is rejected at the **first** limiter that denies it: @@ -95,23 +113,37 @@ Each rejection sets a distinct reason on the plugin context so you can tell whic | Per-key | `per-key rate limit exceeded` | | Per-user | `per-user rate limit exceeded` | -Requests with no API key in metadata skip the per-key check; requests with an empty `user` field skip the per-user check. The three limiters are independent — configure any combination. +Requests with no API key in metadata skip the per-key check; requests with an empty `user` field skip the per-user check. The three limiters are independent — configure any combination. See the [rate-limit plugin reference](/plugins/rate-limit) for the full settings table and failure-mode details. -## Important: `key_rpm` is inert in the bare open-source gateway +## `key_rpm` is keyed on the authenticated credential -:::warning -In the bare open-source gateway, **nothing populates `pctx.Metadata["api_key"]`**. The per-key limiter only fires when that metadata is present, so `key_rpm` (and any per-key budgets) are effectively **inert when self-hosting the OSS gateway as-is** — every request simply skips the per-key check. +`key_rpm` reads `pctx.Metadata["api_key"]`, which the gateway itself populates on every authenticated request — no Ferro Labs Managed layer or custom embedding host required. When a caller authenticates with a bearer API key or a dashboard session, the gateway copies that credential's opaque, non-secret ID (never the raw key) into the plugin context, and `key_rpm` scopes its bucket to that ID. A session inherits the bucket of the API key it was minted from, so re-authenticating cannot reset a caller's limit. -`key_rpm` becomes active only when the embedding host injects an `api_key` into the plugin context metadata. **Ferro Labs Managed** does this as part of its credential/tenant layer; a custom embedding host can do the same. If you are self-hosting and your per-key limits appear to do nothing, this is why — it is expected behavior, not a misconfiguration. +The check is skipped only for requests that carry no authenticated credential at all — for example when `ALLOW_UNAUTHENTICATED_PROXY=true` is set for local development. In that case `key_rpm` has nothing to key on and every request falls through to the global limiter alone. -`user_rpm` has no such dependency: it keys on the request's `user` field, which any OpenAI-compatible client can send, so it works out of the box. -::: +`user_rpm` has no such dependency: it keys on the request's `user` field, which any OpenAI-compatible client can send, so it works identically with or without authentication. + +## Other 429s and the 402 budget response + +Two more rejections are rate-shaped but come from settings outside this page: + +- **`provider_saturated` (429)** — when a target's `targets[].concurrency` limit and its queue are both full, the gateway sheds the request with `429 Too Many Requests` and code `provider_saturated`, carrying the same `Retry-After: 1`. This bounds in-flight requests *per provider target*, independent of both layers above — see [Configuration](/getting-started/configuration) for `targets[].concurrency`. +- **Budget exhaustion (402, not 429)** — the [`budget` plugin](/plugins/budget) rejects an over-cap request with `402 Payment Required` and code `insufficient_quota`, and deliberately carries **no** `Retry-After` header. A spend cap clears on cost roll-off or an explicit reset, never on a timer, so a `429` there would invite a client to retry once a second forever against an answer that cannot change until the cap resets. ## Choosing a layer | Goal | Use | |---|---| -| Coarse abuse protection at the edge, per client IP | Per-IP HTTP middleware (`RATE_LIMIT_RPS`) | +| Coarse abuse protection at the edge, per client IP | Per-IP HTTP middleware (`RATE_LIMIT_RPS`, on by default) | | A ceiling on total throughput to your providers | Plugin `requests_per_second` / `burst` | -| Per-tenant fairness by API key | Plugin `key_rpm` (requires injected `api_key` metadata — Ferro Labs Managed or a custom host) | +| Per-tenant fairness by API key | Plugin `key_rpm` (keyed on the authenticated credential automatically) | | Per-end-user fairness by user ID | Plugin `user_rpm` (send the `user` field on requests) | +| Protect one provider target from overload | `targets[].concurrency` → `provider_saturated` 429 | +| Cap total spend rather than request rate | [`budget` plugin](/plugins/budget) → `402 insufficient_quota` | + +## Related + +- [rate-limit plugin reference](/plugins/rate-limit) +- [budget plugin](/plugins/budget) +- [Configuration](/getting-started/configuration) +- [Server settings](/operations/server-settings) diff --git a/docs/guides/routing-policies.mdx b/docs/guides/routing-policies.mdx deleted file mode 100644 index 1138f5a..0000000 --- a/docs/guides/routing-policies.mdx +++ /dev/null @@ -1,247 +0,0 @@ ---- -title: Routing policies -description: "All 8 LLM routing strategies in the Ferro Labs AI Gateway — single, fallback, weighted, conditional, least-latency, cost-optimized, content-based, and A/B test routing for multi-provider AI traffic." -keywords: [LLM routing strategies, AI fallback routing, weighted LLM routing, cost-optimized AI routing, AI load balancing, least-latency routing, content-based routing, ab-test routing] ---- - -Most AI gateways offer 2–3 routing modes. Ferro Labs ships 8 — covering everything from simple single-provider setups to content-aware routing and live A/B testing. Set `strategy.mode` in `config.yaml` to choose one. - -## Single - -Always routes to the first target. Best for single-provider setups or when you want explicit control. - -**Use this when:** you have one provider and want the simplest possible config. - -```yaml -strategy: - mode: single - -targets: - - virtual_key: openai -``` - -:::tip Performance -Single is the lightest strategy — zero overhead beyond the proxy hop. Ideal for latency-sensitive single-provider deployments. -::: - -## Fallback - -Tries targets in order. On failure (error or retryable status code), the next target is attempted with exponential backoff. Use this for high-availability setups. - -**Use this when:** uptime matters more than anything — your chatbot must always respond, even if the primary provider is down. - -```yaml -strategy: - mode: fallback - -targets: - - virtual_key: openai - retry: - attempts: 3 - on_status_codes: [429, 502, 503] - initial_backoff_ms: 100 - - virtual_key: anthropic - retry: - attempts: 2 - - virtual_key: gemini -``` - -If all targets fail, the last error is returned to the client. - -:::tip Pro tip -Combine fallback with circuit breakers to skip providers that are consistently failing, rather than waiting for retries to timeout on every request. -::: - -## Weighted load balancing - -Distributes requests across targets by weight. Weights are relative — a weight of `70` and `30` sends 70% to the first target and 30% to the second. - -**Use this when:** you want to spread load across providers for cost or capacity reasons. - -```yaml -strategy: - mode: loadbalance - -targets: - - virtual_key: openai - weight: 70 - - virtual_key: anthropic - weight: 30 -``` - -Only targets that support the requested model are candidates for selection. - -:::tip Performance -Weight evaluation adds negligible overhead — a single random number generation per request. Equivalent to single-strategy latency for practical purposes. -::: - -## Conditional - -Evaluates rules in order. The first matching rule determines the target. Each rule has a `key` (the request field to inspect, e.g. `model`), a `value` (the exact value to match), and a `target_key` (the target to route to). - -**Use this when:** different models should route to specific providers — e.g., all GPT models to OpenAI, all Claude models to Anthropic. - -```yaml -strategy: - mode: conditional - conditions: - - key: model - value: gpt-4o - target_key: openai - - key: model - value: gpt-4o-mini - target_key: openai - - key: model - value: claude-3-5-sonnet-20241022 - target_key: anthropic - - key: model - value: gemini-1.5-flash - target_key: gemini - -targets: - - virtual_key: openai - - virtual_key: anthropic - - virtual_key: gemini -``` - -If no rule matches, the request falls through to the first target. - -:::tip Pro tip -Conditional routing pairs well with [model aliases](/getting-started/configuration#model-aliases). Alias `smart` → `claude-3-5-sonnet-20241022` (aliases resolve before routing), then add a conditional rule for `key: model, value: claude-3-5-sonnet-20241022`. -::: - -## Least-latency - -Routes to the target with the lowest P50 latency as measured by a rolling latency tracker. On a cold start (no latency data yet) it picks a target randomly. - -**Use this when:** you have multiple fast providers and want to minimise time-to-first-token automatically. - -```yaml -strategy: - mode: least-latency - -targets: - - virtual_key: openai - - virtual_key: groq - - virtual_key: anthropic -``` - -:::tip Performance -Adds a mutex read on the rolling latency map per request — typically under 1µs. The latency tracker updates asynchronously after each response, so it does not add to request latency. -::: - -## Cost-optimized - -Uses the built-in model catalog (2,500+ entries with pricing data) to estimate the input token cost for each target, then routes to the cheapest compatible provider. Falls back to the first compatible target if cost data is unavailable. - -**Use this when:** you want to minimize spend without manually choosing models — let the catalog handle it. - -```yaml -strategy: - mode: cost-optimized - unpriced_strategy: fallback # fallback (default) | skip | allow - -targets: - - virtual_key: openai - - virtual_key: together - - virtual_key: deepseek - - virtual_key: gemini -``` - -Cost ranking uses input/prompt tokens only, estimated from the request via a ~4 chars/token heuristic, priced against the catalog's `input_per_m_tokens` field (USD per 1M prompt tokens). The target with the lowest estimated cost for the matched model wins. Pricing comes from the [model-catalog](https://github.com/ferro-labs/model-catalog) (a remote release with an embedded fallback, refreshed every 24h, overridable via the `FERRO_MODEL_CATALOG_URL` env var). `unpriced_strategy` controls how targets with no pricing data are handled: `fallback` prefers priced candidates then the first compatible unpriced target, `skip` rejects unpriced candidates, and `allow` treats missing prices as zero cost. - -:::tip Pro tip -Combine cost-optimized with fallback by adding `retry` to each target — if the cheapest provider fails, the gateway retries with the next cheapest. -::: - -## Content-based - -Routes based on the content of the user's messages. Rules are evaluated in order; the first match wins. If no rule matches, the request falls through to the first target. - -**Use this when:** different types of queries should go to different specialized models — code to a coding model, translation to a translation service, general chat to a cost-efficient default. - -Three condition types are supported: - -| Type | Behavior | -|---|---| -| `prompt_contains` | Case-insensitive substring match on any user message | -| `prompt_not_contains` | Matches when NO user message contains the value | -| `prompt_regex` | Go regular-expression match on any user message | - -Regex patterns are compiled at gateway startup. An invalid regex causes a startup error — there is no silent misrouting. - -```yaml -strategy: - mode: content-based - content_conditions: - - type: prompt_contains - value: "translate" - target_key: deepl-provider - - type: prompt_regex - value: "(?i)(code|function|class|def |import )" - target_key: openai - - type: prompt_contains - value: "summarize" - target_key: anthropic - -targets: - - virtual_key: deepl-provider - - virtual_key: openai - - virtual_key: anthropic -``` - -:::tip Performance -Substring matching (`prompt_contains`) is near-zero cost. Regex matching adds overhead proportional to pattern complexity, but patterns are pre-compiled at startup so the hot path is a single `regexp.MatchString` call. -::: - -## A/B test - -Splits traffic across variants by configured weights. Each variant carries a `label` field, which is currently emitted only to the gateway's DEBUG logs. - -**Use this when:** you want to compare quality, latency, or cost between two providers on live traffic without client-side changes. - -```yaml -strategy: - mode: ab-test - ab_variants: - - target_key: openai - weight: 70 - label: control - - target_key: anthropic - weight: 30 - label: challenger - -targets: - - virtual_key: openai - - virtual_key: anthropic -``` - -Weights are relative — `70` and `30` send 70% of traffic to `openai` and 30% to `anthropic`. If a weight is `0`, the variant is treated as weight `1` (equal distribution with remaining variants). Negative weights are rejected at gateway startup. - -The `label` field is currently visible only in the gateway's DEBUG logs — it is not written to events or a queryable log column. - -:::tip Pro tip -Combine A/B test with the `request-logger` plugin persisting to Postgres to compare aggregate latency and token usage across live traffic. Per-variant attribution currently requires reading the variant `label` from the gateway's DEBUG logs. -::: - -## Combining strategies with circuit breakers - -All strategies respect per-target circuit breakers. A target whose circuit breaker is open is excluded from selection. - -```yaml -targets: - - virtual_key: openai - circuit_breaker: - failure_threshold: 5 - success_threshold: 2 - timeout: "30s" -``` - -The circuit breaker opens after `failure_threshold` consecutive failures, stays open for `timeout`, then enters half-open state where it allows one probe request. After `success_threshold` successes it closes again. - -## Related pages - -- [Configuration reference](/getting-started/configuration) — full YAML reference for all strategy modes -- [Use cases](/guides/use-cases) — recipe-style configurations for common scenarios -- [Benchmarks](/benchmarks) — performance data for different routing strategies -- [Plugins](/guides/plugins) — combine routing with safety and observability plugins diff --git a/docs/guides/troubleshooting.mdx b/docs/guides/troubleshooting.mdx index 5093ef2..6eee73b 100644 --- a/docs/guides/troubleshooting.mdx +++ b/docs/guides/troubleshooting.mdx @@ -1,7 +1,7 @@ --- title: Troubleshooting -description: "Solutions for common Ferro Labs AI Gateway issues — provider keys, circuit breakers, streaming, MCP timeouts, rate limiting, config reload, Docker health checks, and Prometheus scraping." -keywords: [AI gateway troubleshooting, LLM proxy debugging, gateway error codes, AI gateway FAQ, gateway connection issues] +description: "Diagnose Ferro Labs AI Gateway errors: model_not_found vs upstream_unavailable, circuit breakers, MCP stdio tool calls, A/B weights, and Prometheus scraping." +keywords: [AI gateway troubleshooting, LLM proxy debugging, gateway error codes, circuit breaker open, MCP stdio debugging, Prometheus 401, insufficient_quota] --- import Head from '@docusaurus/Head'; @@ -11,13 +11,31 @@ import Head from '@docusaurus/Head'; "@context": "https://schema.org", "@type": "TechArticle", "headline": "Ferro Labs AI Gateway Troubleshooting Guide", - "description": "Solutions for common Ferro Labs AI Gateway issues — provider keys, circuit breakers, streaming, MCP timeouts, rate limiting, config reload, Docker health checks, and Prometheus scraping.", + "description": "Diagnose Ferro Labs AI Gateway errors: model_not_found vs upstream_unavailable, circuit breakers, MCP stdio tool calls, A/B weights, and Prometheus scraping.", "url": "https://docs.ferrolabs.ai/guides/troubleshooting/" })} This page covers the most common issues encountered when running the Ferro Labs AI Gateway and how to resolve them. +## Reading the status code first + +Before chasing a symptom below, the HTTP status and `error.code` the gateway returned usually says which failure class you're in. All four routed surfaces (chat, streaming, embeddings, images) share one classifier, so the mapping is the same everywhere: + +| Status | `error.code` | What actually happened | +| --- | --- | --- | +| `404` | `model_not_found` | No configured target names this model. The provider was **never called** — this is a routing/config problem, not an outage. | +| `503` | `upstream_unavailable` | The target's circuit breaker is **open**. The gateway refused the call without contacting the provider. | +| `502` | `upstream_error` | The gateway called the provider and got a `5xx`, a connection failure, or another status it doesn't have a more specific answer for. | +| `502` | `upstream_auth_error` | The provider returned `401`/`403`. This is the gateway's own credential being rejected, not yours — reported as an upstream fault so your key isn't blamed. | +| `504` | `upstream_timeout` / `gateway_timeout` | The provider didn't respond in time, or the gateway's own `request_timeout` fired first. | +| `429` | `rate_limit_exceeded` | The per-IP limiter, the `rate-limit` plugin, or the provider's own `429` passed through. | +| `429` | `provider_saturated` | `targets[].concurrency` is full — this is backpressure, not a failure. | +| `402` | `insufficient_quota` | The `budget` plugin: this API key is at or over `spend_limit_usd`. | +| `400` | `request_rejected` | A `before_request` guardrail (word-filter, max-token) denied the request. | + +The rest of this page walks through the likely causes behind each of these. + ## Provider key not picked up at startup **Symptom:** The gateway starts but returns `401 Unauthorized` for every request to a provider you configured. @@ -30,13 +48,13 @@ Check whether the variable is available inside the running container: ```bash # Docker -docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' ferrogw | grep OPENAI +docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' gateway | grep OPENAI # Local process env | grep OPENAI_API_KEY ``` -Provider credentials are read from environment variables (e.g. `OPENAI_API_KEY`), not from `config.yaml`. Make sure each target's `virtual_key` names a provider whose key env var is set: +Provider credentials are read from environment variables (e.g. `OPENAI_API_KEY`), not from `config.yaml`. Each target's `virtual_key` names a provider — a provider is only registered when that provider's key env var is present: ```yaml targets: @@ -44,48 +62,54 @@ targets: - virtual_key: anthropic # requires ANTHROPIC_API_KEY in the environment ``` +A target naming an unregistered provider is a startup **warning**, not a crash — the gateway keeps serving what it can. Check the startup log for `WARN` lines naming unroutable targets, and `GET /readyz` for `routable: false` on the affected target. + :::warning -Never hard-code API keys. Provider keys are supplied only via environment variables (`OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, …); inject them through your orchestrator or `.env` file. +Never hard-code API keys. Provider keys are supplied only via environment variables; inject them through your orchestrator or `.env` file. ::: --- ## Circuit breaker opens immediately -**Symptom:** After a single failed request the target is excluded from routing and the gateway returns errors or falls through to the next target. +**Symptom:** After a handful of failed requests the target is excluded from routing and the gateway returns `503 upstream_unavailable` or silently shifts traffic to another target. -**Likely cause:** `failure_threshold` is set to `1`, so one failure trips the breaker. Alternatively, the upstream provider is genuinely down. +**Likely cause:** `failure_threshold` is set too low, or the upstream provider is genuinely failing. **Fix:** -First, check upstream health: +Check circuit state directly rather than guessing: ```bash -curl -s http://localhost:8080/health | jq . +curl -s http://localhost:8080/health | jq '.providers' ``` -If the provider is healthy, raise the threshold: +`gateway_circuit_breaker_state{provider=""}` on `/metrics` is the more precise signal — a series only exists for targets that actually have a breaker configured, so its absence means no breaker is set up at all for that target. + +If the provider is genuinely healthy, raise the threshold: ```yaml targets: - virtual_key: openai circuit_breaker: - failure_threshold: 5 # require 5 consecutive failures before opening - success_threshold: 2 - timeout: "30s" + failure_threshold: 5 # default; require 5 consecutive failures before opening + success_threshold: 2 # default is 1; require 2 successes in half-open to close + timeout: "30s" # default; how long the breaker stays open before probing again ``` :::tip -Set `failure_threshold` to at least `3` in production to avoid flapping on transient errors. +Not every failure counts toward opening the breaker. A `429`, a client disconnect, an unsupported-parameter rejection, and a request shed by `targets[].concurrency` are all excluded — none of them is evidence the upstream is unhealthy. A `5xx`, a connection failure, and the streaming idle bound elapsing all do count. ::: +**A breaker is scoped to the `virtual_key`, not to an endpoint.** One breaker per target is shared across chat, streaming, embeddings, and image generation. A provider that only fails `/v1/embeddings` (a moved route, a per-surface auth failure) still trips the shared breaker and takes `/v1/chat/completions` down with it for that target, even though no chat request ever failed. If chat traffic breaks with no visible chat-side errors, check `GET /admin/logs?provider=` across every surface, not just the one you're calling. + --- ## Streaming responses truncated **Symptom:** Server-sent event (SSE) streams cut off before the model finishes generating. The client receives a partial response. -**Likely cause:** A reverse proxy or load balancer between the client and the gateway is timing out before the stream completes. +**Likely cause:** A reverse proxy or load balancer between the client and the gateway is timing out before the stream completes. `request_timeout`, if you have one configured, does **not** apply here — streaming is explicitly exempt from it. **Fix:** @@ -102,44 +126,52 @@ location /v1/ { } ``` -If the gateway itself is timing the request out, check your target-level timeout configuration. Streaming requests to large models can take 60 seconds or more. +300s is not an arbitrary number here — it matches the gateway's own **stream idle timeout**, a fixed 5-minute bound built into the transport layer (not a `config.yaml` key). If no new chunk arrives from the provider for 5 minutes, the gateway itself cuts the stream and that counts as a failure toward the target's circuit breaker. If you see truncations well under 5 minutes, the timeout is almost always in front of the gateway, not inside it. --- -## MCP server connection timeout +## MCP tool calls not firing, or timing out -**Symptom:** Requests that should trigger tool calls return a plain text response, or the gateway logs `mcp connection timeout`. +**Symptom:** A request that should trigger a tool call returns a plain-text response with no `tool_calls`, or `GET /readyz` reports an MCP server as `ready: false`. -**Likely cause:** The MCP server URL is wrong, the server is not running, or a firewall is blocking the connection. +MCP has two failure shapes worth telling apart: the gateway never offered the model any tools, or it offered them and a specific server is broken. -**Fix:** +### Tools were never offered to the model -Check the gateway logs for MCP-related errors: +**Likely cause:** MCP tools are only injected when the incoming request carries **no `tools` of its own**. If your client SDK attaches even an empty `tools: []`, or its own function-calling definitions, the gateway leaves the request alone and passes it straight through — this is deliberate, so a caller's own tool contract is never silently mixed with the gateway's. -```bash -docker logs ferrogw 2>&1 | grep -i mcp -``` +**Fix:** Confirm the request body has no `tools` field, and check `GET /metrics` for `gateway_mcp_server_up{server_name=...}` — a value of `1` means the server itself is fine and the problem is upstream of MCP, in the request shape. -Test connectivity from the gateway's network: +### A specific server won't come up -```bash -# From inside the container -docker exec ferrogw curl -sf http://mcp-server:3001/mcp -``` +**Likely cause, stdio (`command:`) servers:** an unresolved `${VAR}` in `env:` or `headers:` fails the client at construction — the server is recorded as unready rather than omitted, and `GET /readyz` shows it under `mcp_servers` with `ready: false`. Since `/readyz` is unauthenticated, it never names the failure; check `GET /admin/health` (bearer token, `read_only` or `admin` scope) for the actual resolution error. -Verify the URL and timeout in `config.yaml`: +**Likely cause, stdio subprocess crashed after startup:** the gateway drains the subprocess's stderr and treats its close as a death suspicion, confirmed by an MCP ping before withdrawing the server's tools. Set `LOG_LEVEL=debug` to see the drained stderr lines — they log at Debug, not Error, since stderr output is not itself an error per the MCP spec. + +**Likely cause, HTTP (`url:`) server unreachable:** post-handshake death is **not detected** for HTTP servers — an unreachable one keeps reporting `ready: true` with its tools still advertised, and every call to it simply fails per request. `required: true` cannot pull an instance out of rotation for a dead HTTP server; it only helps if the server was never reachable in the first place. + +**Fix:** ```yaml mcp_servers: + # stdio: launched as a subprocess at gateway startup, lives for the process lifetime - name: filesystem - url: "http://mcp-server:3001/mcp" # must be reachable from the gateway - timeout_seconds: 15 # increase if the server is slow to respond - max_call_depth: 5 + command: npx + args: ["-y", "@modelcontextprotocol/server-filesystem", "/data"] + env: # the subprocess inherits NO gateway environment — + SOME_TOKEN: ${SOME_TOKEN} # only PATH/HOME/LANG/TMPDIR plus exactly what's listed here + timeout_seconds: 30 # per tool call; default 30 + required: false # default; true gates /readyz on THIS server alone + + # Streamable HTTP: the gateway connects to a running endpoint + - name: database + url: "https://mcp-db.internal/mcp" + headers: + Authorization: "Bearer ${MCP_DB_TOKEN}" + allowed_tools: ["query_readonly", "list_tables"] # empty = all discovered tools ``` -:::tip -If the MCP server runs in a separate Docker Compose service, make sure both services are on the same Docker network. -::: +Set `mcp_servers[].required: true` only for a server the deployment genuinely cannot serve without — an unready required server takes `/readyz` to `503`, gating **all** traffic through the instance, including requests that use no tools at all. --- @@ -147,11 +179,11 @@ If the MCP server runs in a separate Docker Compose service, make sure both serv **Symptom:** Clients receive `429 Too Many Requests` well below expected traffic levels. -**Likely cause:** The `burst` value is too low, or the global rate limit is being confused with the per-key limit. The global bucket drains across all clients combined. +**Likely cause:** Two independent limiters exist, and it's easy to tune the wrong one. The per-IP HTTP limiter (`RATE_LIMIT_RPS` / `RATE_LIMIT_BURST` env vars, default 20 rps / burst 40) applies to every request before it reaches any plugin. The `rate-limit` **plugin** is a separate global + per-key + per-user token bucket configured under `plugins:`. **Fix:** -Review your rate-limit plugin configuration. The global `requests_per_second` applies to **all traffic**, while `key_rpm` applies per API key: +Review your rate-limit plugin configuration. The global `requests_per_second` applies to **all traffic**, `key_rpm` applies per API key, and `user_rpm` applies per `Request.User`: ```yaml plugins: @@ -161,55 +193,41 @@ plugins: enabled: true config: requests_per_second: 100 # global: 100 req/s across all clients - burst: 200 # allow short bursts up to 200 + burst: 100 # burst capacity; defaults to requests_per_second key_rpm: 60 # per-key: max 60 requests per minute ``` -If you only want per-key limits and no global cap, omit `requests_per_second` and `burst`: - -```yaml -plugins: - - name: rate-limit - type: guardrail - stage: before_request - enabled: true - config: - key_rpm: 60 -``` +:::warning +`0` means the opposite thing on each limiter. `RATE_LIMIT_RPS=0` **disables** the per-IP HTTP limiter. `requests_per_second: 0` on the `rate-limit` plugin is a **load error** — a rate of zero would blackhole all traffic, so `ferrogw validate` rejects it at startup. To turn the plugin off, set `enabled: false` instead of a zero rate. +::: -Rate checks execute in order: **global, then per-key, then per-user**. The first exceeded limit triggers the `429`. +Check order is: global bucket, then per-key, then per-user — the first exceeded limit wins. A gateway-decided `429` (from either limiter) carries a `Retry-After: 1` header; if your client isn't backing off, confirm it reads that header rather than guessing its own schedule. --- ## Config reload not taking effect -**Symptom:** You edited `config.yaml` but the gateway behavior has not changed. +**Symptom:** You edited `config.yaml` and restarted the gateway, but the running behavior hasn't changed. -**Likely cause:** The `GATEWAY_CONFIG` environment variable points to a different file, or the edited file has a YAML syntax error that causes a silent reload failure. +**Likely cause:** The gateway resolves its active config from **three sources, in order of precedence**: a config previously written through `PUT /admin/config` (kept in `CONFIG_STORE_BACKEND`), then the file at `GATEWAY_CONFIG`, then a built-in default. There is no filesystem watch — editing the file only ever takes effect on the **next process start**, and even then, a persisted store config wins over it every time. **Fix:** -Confirm which file the gateway is loading: +Check the startup log line `active config resolved` — it names the winning `source` (`store`, `file`, or `defaults`). If a `CONFIG_STORE_BACKEND` is configured and something was ever pushed via `PUT /admin/config` or the dashboard, that stored config **always** wins over your file, and you'll additionally see a `WARN` log: `config file superseded by the persisted config`. ```bash +# Confirm which file GATEWAY_CONFIG points at echo $GATEWAY_CONFIG -# Should print the path to the config file you edited -``` -Validate the YAML before reloading: - -```bash -# Quick syntax check (requires yq or python) -yq eval '.' config.yaml > /dev/null && echo "YAML OK" || echo "YAML ERROR" - -# Or with Python -python3 -c "import yaml; yaml.safe_load(open('config.yaml'))" +# Discard the stored config so the file applies again on next restart +curl -X DELETE http://localhost:8080/admin/config \ + -H "Authorization: Bearer $MASTER_KEY" ``` -After fixing any syntax issues, restart the gateway: +Validate the YAML before restarting — decoding is strict (unknown keys are rejected) and a syntax error or unknown field exits the process rather than falling back silently: ```bash -docker restart ferrogw +ferrogw validate config.yaml ``` --- @@ -222,7 +240,7 @@ docker restart ferrogw **Fix:** -If you use the `response-cache` plugin, set a maximum entry count: +If you use the `response-cache` plugin, cap its size. It's a **multi-stage** plugin: it must be listed at both `before_request` (serve a hit) and `after_request` (store a new entry) with **byte-for-byte identical config**, or the gateway refuses to start. A single-stage entry silently never stores anything. ```yaml plugins: @@ -231,15 +249,22 @@ plugins: stage: before_request enabled: true config: - max_entries: 5000 # cap the cache to prevent unbounded growth - max_age: 300 # entry TTL in seconds + max_age: 300 # TTL in seconds; default 300 + max_entries: 1000 # LRU capacity; default 1000. <= 0 disables storing + - name: response-cache + type: transform + stage: after_request + enabled: true + config: + max_age: 300 + max_entries: 1000 ``` -Profile the gateway with pprof to identify the source of allocations: +Profile the gateway with pprof to identify the source of allocations (requires `ENABLE_PPROF=true`; the routes are `admin`-scope only): ```bash -# Requires LOG_LEVEL=debug or pprof enabled -curl -s http://localhost:8080/debug/pprof/heap > heap.out +curl -s -H "Authorization: Bearer $MASTER_KEY" \ + http://localhost:8080/debug/pprof/heap > heap.out go tool pprof heap.out ``` @@ -253,135 +278,105 @@ In Docker deployments, set memory limits on the container (`--memory=2g`) so an **Symptom:** Docker reports the gateway container as `unhealthy` even though it is processing requests. -**Likely cause:** The healthcheck is hitting the wrong port or path. +**Likely cause:** The official image (`ghcr.io/ferro-labs/ai-gateway` — a single multi-platform manifest, no separate `-amd64`/`-arm64` tag) ships its own `HEALTHCHECK` **baked in at build time**: `wget -qO- http://localhost:8080/readyz`. It's `/readyz` rather than `/livez` on purpose — a liveness check alone would keep routing traffic to an instance whose targets are all unroutable. If you set `PORT` to anything other than `8080` without also overriding the healthcheck, the baked-in check keeps probing the old port, gets a connection refused, and the container reports `unhealthy` even though the gateway itself is fine on the new port. **Fix:** -The gateway exposes its health endpoint at `/health` on the configured `PORT` (default `8080`). Make sure your `docker-compose.yml` or `Dockerfile` healthcheck matches: +If you're not overriding the healthcheck, you don't need to configure one — the image already does the right thing. If you *do* define a `healthcheck:` block (in Compose or a derived `Dockerfile`), it fully replaces the image's built-in one, so keep the port in sync with `PORT`: ```yaml services: gateway: - image: ghcr.io/ferrolabs/ferrogw:latest + image: ghcr.io/ferro-labs/ai-gateway:latest ports: - "8080:8080" + environment: + - PORT=8080 # keep this in sync with the healthcheck URL below healthcheck: - test: ["CMD", "curl", "-sf", "http://localhost:8080/health"] + test: ["CMD", "wget", "--spider", "-q", "http://localhost:8080/readyz"] interval: 10s timeout: 5s retries: 3 start_period: 5s ``` -If you changed the port via the `PORT` environment variable, update the healthcheck URL to match: - -```yaml -environment: - PORT: "9090" -healthcheck: - test: ["CMD", "curl", "-sf", "http://localhost:9090/health"] -``` +`/health` is a valid alternative if you specifically want the deep diagnostic payload (per-provider circuit state, model counts), but it always answers `200` regardless of whether any target is actually routable — it isn't the "should traffic reach this instance" signal that `/readyz` is. --- ## Prometheus scrape returning empty -**Symptom:** Prometheus shows no metrics for the gateway, or `curl` to the metrics endpoint returns an empty body. +**Symptom:** Prometheus shows no metrics for the gateway (`up == 0`), or `curl` to the metrics endpoint returns `401`. -**Likely cause:** The metrics endpoint is not enabled, or Prometheus is scraping the wrong port. +**Likely cause:** `/metrics` requires a bearer token carrying the `read_only` or `admin` scope, same as every other admin read route. A scrape config with no `Authorization` header gets `401`, and Prometheus reports the target as down — this, not a disabled endpoint, is by far the most common cause. **Fix:** -Verify the metrics endpoint is responding: +Verify the endpoint requires auth: ```bash -curl -s http://localhost:8080/metrics | head -20 +curl -i http://localhost:8080/metrics +# HTTP/1.1 401 Unauthorized — confirms this, not a routing/port problem ``` -If you get a `404`, confirm that metrics are enabled in your server settings. The gateway exposes Prometheus metrics at `GET /metrics` by default on the same port as the API. +Mint a dedicated `read_only` key for scraping — don't reuse `MASTER_KEY` in a Prometheus config file that might leak: -Check your `prometheus.yml` targets: +```bash +curl -X POST http://localhost:8080/admin/keys \ + -H "Authorization: Bearer $MASTER_KEY" \ + -H "Content-Type: application/json" \ + -d '{"name": "prometheus-scraper", "scopes": ["read_only"]}' +# -> {"id": "...", "key": "fgw_...", "scopes": ["read_only"], ...} +``` ```yaml scrape_configs: - - job_name: ferrogw + - job_name: ferro-ai-gateway + metrics_path: /metrics + authorization: + type: Bearer + credentials: fgw_the_key_returned_above static_configs: - - targets: ["gateway-host:8080"] # must match the gateway's PORT - scrape_interval: 15s + - targets: ["gateway-host:8080"] ``` -:::tip -If the gateway runs inside Docker Compose and Prometheus runs in the same stack, use the service name as the host: `targets: ["gateway:8080"]`. -::: +See [Monitoring and operations](/operations/monitoring) for the full metrics reference and alerting rules. --- -## 502 Bad Gateway from all providers +## Model not found error -**Symptom:** Every request returns `502 Bad Gateway` regardless of the target provider. +**Symptom:** The gateway returns `404` with `error.code: "model_not_found"`, even though you have a target configured for that provider. -**Likely cause:** All circuit breakers are open because every upstream provider is failing (or was recently failing). +**Likely cause:** No configured target's routing index (catalog + live discovery + `targets[].models`) contains the exact model id the request named. This check runs **before** any plugin — a request that fails it never spends a rate-limit token or a budget dollar. **Fix:** -Check health to see provider status: +List models the gateway currently believes it can route: ```bash -curl -s http://localhost:8080/health | jq . +curl -s http://localhost:8080/v1/models \ + -H "Authorization: Bearer $API_KEY" | jq '.data[].id' ``` -If the upstream providers have recovered but circuit breakers are still open, they will close automatically after the configured `timeout` period. You can speed this up by restarting the gateway: +If a name doesn't need mapping — you just want a friendly alias — use `aliases`, resolved before routing: -```bash -docker restart ferrogw +```yaml +aliases: + our-default: gpt-4o-mini + our-smart: claude-sonnet-4-20250514 ``` -To prevent all breakers from opening simultaneously, stagger your `failure_threshold` and `timeout` values across targets: +If a target genuinely serves a model neither the catalog nor live discovery knows about yet (a brand-new id, a preview name, a self-hosted deployment), declare it on the target instead — this is additive only, and never restricts what a target already serves: ```yaml targets: - - virtual_key: openai - circuit_breaker: - failure_threshold: 5 - timeout: "30s" - - virtual_key: anthropic - circuit_breaker: - failure_threshold: 3 - timeout: "20s" - virtual_key: gemini - circuit_breaker: - failure_threshold: 5 - timeout: "45s" + models: + - gemini-2.5-flash-preview # exact id only; wildcards are rejected at load ``` ---- - -## Model not found error - -**Symptom:** The gateway returns an error like `model "gpt4o" not found` even though you have an OpenAI target configured. - -**Likely cause:** The model name in the request does not match any entry in the built-in catalog or your configured models list. Model names are exact-match (e.g. `gpt-4o`, not `gpt4o`). - -**Fix:** - -List available models through the gateway: - -```bash -curl -s http://localhost:8080/v1/models \ - -H "Authorization: Bearer $API_KEY" | jq '.data[].id' -``` - -If you need to map a custom name to a real model, use model aliases in your config: - -```yaml -model_aliases: - - alias: "our-default" - model: "gpt-4o-mini" - target_key: openai - - alias: "our-smart" - model: "claude-sonnet-4-20250514" - target_key: anthropic -``` +Remember that under `single`, `conditional`, and `content-based` modes, only the **named** target is ever attempted — a model owned solely by a *different* configured target is still a 404 under those modes, however many targets the config lists overall. --- @@ -389,20 +384,11 @@ model_aliases: **Symptom:** Requests that should match a `prompt_regex` rule are falling through to the default target instead. -**Likely cause:** The regex pattern is not matching due to case sensitivity, or the pattern has a syntax error that prevented compilation. +**Likely cause:** The regex pattern is not matching due to case sensitivity, or `content_conditions[].type` is not one of the values the gateway accepts. **Fix:** -Regex patterns are compiled at gateway startup. An invalid regex causes a **startup error** (not a silent failure). Check gateway startup logs for compilation errors. - -If the gateway started successfully but routing is wrong, test your regex independently: - -```bash -# Test a Go-compatible regex -echo "Write a Python function to sort a list" | grep -P '(?i)(code|function|class|def |import )' -``` - -Remember that the gateway uses Go regular expressions. Use `(?i)` at the start for case-insensitive matching: +`content_conditions[].type` is a closed set (`prompt_contains`, `prompt_not_contains`, `prompt_regex`) validated at startup — an unknown type or an uncompilable regex is a **startup error**, not a silent fallback. If the gateway started successfully, the pattern compiled; the issue is what it matches against. ```yaml strategy: @@ -413,23 +399,23 @@ strategy: target_key: deepseek ``` +The gateway uses Go regular expressions — use `(?i)` at the start for case-insensitive matching, as above. + --- ## A/B test weights not reflecting expected distribution -**Symptom:** You configured an 80/20 split but after 50 requests you see a 60/40 ratio. +**Symptom:** You configured an 80/20 split but after 50 requests you see a 60/40 ratio, or a variant seems to get no traffic at all. -**Likely cause:** With small sample sizes, random weighted selection naturally deviates from the configured weights. This is expected statistical variance, not a bug. +**Likely cause:** Either normal statistical variance at a small sample size, or a misunderstanding of what `weight: 0` does. **Fix:** -Weight normalization works as follows: weights are relative, so `80` and `20` produce an 80/20 split. Over a large number of requests (1,000+) the actual distribution converges toward the configured ratio. +Weights are relative: `80` and `20` produce an 80/20 split, converging as request volume grows. A few things to check: -A few things to check: - -- **Zero weights**: If a variant has weight `0`, it is treated as weight `1` and receives roughly equal traffic with other zero-weight variants. This is by design to prevent accidentally silencing a variant. -- **Circuit breakers**: If the target backing one variant has its circuit breaker open, all traffic goes to the remaining variant. -- **Sample size**: At 100 requests with an 80/20 split, a 70/30 or 90/10 actual split is within normal variance. Collect at least 1,000 requests before evaluating. +- **`weight: 0` means zero traffic — a hard drain, not "treated as weight 1".** This is the deliberate way to stop routing to a variant without removing it from the config. A **negative** weight, or a variant set with **every** weight at zero, is rejected at startup (`ferrogw validate` catches it) — it never reaches request time. +- **Circuit breakers**: an open-circuit target is skipped in every routing mode, ab-test included, so its traffic share silently moves to the remaining variant(s). +- **Sample size**: at 100 requests with an 80/20 configured split, a 70/30 or 90/10 observed split is within normal variance. Collect 1,000+ requests before evaluating. ```yaml strategy: @@ -447,43 +433,65 @@ strategy: ## Budget plugin not persisting across restarts -**Symptom:** After restarting the gateway, all API key spend counters reset to zero. +**Symptom:** After restarting the gateway, all API key spend counters reset to zero. A key that was denied with `402 insufficient_quota` before the restart is admitted again immediately after. -**Likely cause:** This is by design. The budget plugin uses an **in-memory** store that resets on every restart. +**Likely cause:** This is by design. The `budget` plugin's spend store is **in-memory** and resets on every restart; it's a soft, session-scoped cap, not durable billing enforcement. **Fix:** -The open-source budget plugin is intended for session-scoped soft limits and development quotas. If you need durable spend tracking that survives restarts: +Make sure both stage entries exist with byte-identical config — `budget` needs `before_request` (check) *and* `after_request` (record spend), sharing state through `store_id`, or the check side never sees what the record side wrote: + +```yaml +plugins: + - name: budget + type: guardrail + stage: before_request + enabled: true + config: + store_id: default + spend_limit_usd: 10.0 + input_per_m_tokens: 3.0 + output_per_m_tokens: 15.0 + - name: budget + type: guardrail + stage: after_request + enabled: true + config: + store_id: default + spend_limit_usd: 10.0 + input_per_m_tokens: 3.0 + output_per_m_tokens: 15.0 +``` + +Two more things worth knowing: an **unauthenticated** request (`ALLOW_UNAUTHENTICATED_PROXY=true`) is never tracked or capped, since spend keys on the credential's `api_key_id`; and `max_keys` (default 10,000) evicts the **lowest-spend** key when full, silently restarting its counter at `$0`. + +If you need durable spend tracking that survives restarts: -- Use [Ferro Labs Managed](https://ferrolabs.ai) for persistent billing enforcement with database-backed spend tracking. -- As a workaround, export spend data via the `/metrics` endpoint before restarting and use your monitoring system for budget alerts. +- Use [Ferro Labs Managed](https://ferrolabs.ai) for database-backed spend tracking. +- As a workaround, export `/metrics` before restarting and alert from your monitoring system. :::warning -Do not rely on the in-memory budget plugin as your only spend control in production. A restart silently resets all limits. Use it as a safety net alongside durable billing in Ferro Labs Managed. +Do not rely on the in-memory budget plugin as your only spend control in production. A restart silently resets every limit. ::: --- ## Request logger not writing to Postgres -**Symptom:** The `request-logger` plugin is enabled but no rows appear in the Postgres `request_logs` table. +**Symptom:** The `request-logger` plugin is enabled but no rows appear in the Postgres request-log table. -**Likely cause:** The connection string (DSN) is wrong, Postgres is not reachable from the gateway, or the database/table does not exist. +**Likely cause:** Persistence is a **process-level** setting, not a per-plugin one — `backend` and `dsn` inside the plugin's `config:` block are obsolete and silently ignored (with a startup warning). The actual target is `REQUEST_LOG_STORE_BACKEND` / `REQUEST_LOG_STORE_DSN`. **Fix:** -Test connectivity from the gateway's environment: +Set the store at the process level: ```bash -# From inside the Docker container -docker exec ferrogw sh -c \ - 'pg_isready -h postgres -p 5432 -U ferro || echo "Postgres unreachable"' - -# Or test with curl/psql from the host -psql "postgres://ferro:ferro_secret@localhost:5432/ferro_logs" -c "SELECT 1;" +REQUEST_LOG_STORE_BACKEND=postgres +REQUEST_LOG_STORE_DSN=postgres://ferro:ferro_secret@postgres:5432/ferro_logs?sslmode=disable ``` -Verify the plugin config matches the actual Postgres host, port, user, and database: +Then enable `persist: true` on the plugin. `request-logger` is **three-stage** — `before_request`, `after_request`, *and* `on_error` — all with identical config. The `on_error` entry isn't optional: a failed request never reaches `after_request`, so without it, failures vanish from `GET /admin/logs` entirely. ```yaml plugins: @@ -494,26 +502,41 @@ plugins: config: level: info persist: true - backend: postgres - dsn: postgres://ferro:ferro_secret@postgres:5432/ferro_logs?sslmode=disable + - name: request-logger + type: logging + stage: after_request + enabled: true + config: + level: info + persist: true + - name: request-logger + type: logging + stage: on_error + enabled: true + config: + level: info + persist: true ``` -:::tip -If you run both the gateway and Postgres in Docker Compose, use the Compose service name (e.g. `postgres`) as the hostname, not `localhost`. -::: - -Check gateway logs for connection errors: +Test connectivity from the gateway's environment, and check gateway logs for a startup warning naming the misconfigured store: ```bash -docker logs ferrogw 2>&1 | grep -i postgres +docker exec gateway sh -c \ + 'pg_isready -h postgres -p 5432 -U ferro || echo "Postgres unreachable"' + +docker logs gateway 2>&1 | grep -i "request log\|postgres" ``` +:::tip +If you run both the gateway and Postgres in Docker Compose, use the Compose service name (e.g. `postgres`) as the hostname, not `localhost`. +::: + --- ## Related pages - [Configuration](/getting-started/configuration) -- [Plugins](/guides/plugins) +- [Plugins](/plugins) - [MCP integration](/guides/mcp) -- [Observability](/guides/observability) +- [Error reference](/api-reference/errors) - [Monitoring and operations](/operations/monitoring) diff --git a/docs/guides/use-cases.mdx b/docs/guides/use-cases.mdx index 7d80881..4498b73 100644 --- a/docs/guides/use-cases.mdx +++ b/docs/guides/use-cases.mdx @@ -1,10 +1,10 @@ --- title: Common Use Cases -description: "Recipe-style AI Gateway configurations for failover, cost optimization, A/B testing, content routing, rate-limited free tiers, and agentic MCP pipelines." -keywords: [AI gateway use cases, LLM failover config, AI cost optimization, A/B test LLM, content-based routing AI, MCP agentic pipeline] +description: "Copy-paste AI Gateway config recipes: failover, cost-optimized and A/B routing, budget caps, self-hosted models, batch jobs, native audio, and MCP agents." +keywords: [AI gateway use cases, LLM failover config, AI cost optimization, A/B test LLM, content-based routing AI, self-hosted LLM routing, batch API gateway, MCP agentic pipeline] --- -Complete, copy-pasteable configurations for the most common deployment patterns. Each recipe includes a full `config.yaml` and a `curl` command you can run immediately. +Complete, copy-pasteable configurations for the most common deployment patterns. Each recipe includes a full `config.yaml` and a `curl` command you can run immediately. Auth is on by default, so every request below carries an `Authorization: Bearer` header — see [Authentication](/guides/auth) for issuing API keys. ## 1. Multi-provider failover for a production chatbot @@ -55,19 +55,20 @@ curl -s http://localhost:8080/v1/chat/completions \ }' ``` -If OpenAI returns a `429` or `5xx`, the gateway automatically retries up to 3 times with exponential backoff, then falls through to Anthropic (translating the request format on the fly), and finally to Gemini. The client sees a single response with no indication of the failover. +If OpenAI returns a `429` or `5xx`, the gateway automatically retries up to 3 times with exponential backoff, then falls through to Anthropic (translating the request format on the fly), and finally to Gemini. `fallback` is the only mode where the pipeline advances to the next target on failure; `retry` on each target (honoured under every mode) is how many times that one target is re-asked first. The client sees a single response with no indication of the failover. --- ## 2. Cost optimization: route to the cheapest compatible model -The `cost-optimized` strategy uses the built-in model catalog (2,500+ entries with pricing data) to estimate the input token cost for each target and routes to the cheapest one. +The `cost-optimized` strategy prices `/` for every compatible target through the built-in model catalog (2,500+ entries with pricing data) and routes to the cheapest one. Register each provider by setting its API key environment variable (`TOGETHER_API_KEY`, `DEEPSEEK_API_KEY`, `GEMINI_API_KEY`, `OPENAI_API_KEY`), then reference them by `virtual_key`. ```yaml title="config.yaml" strategy: mode: cost-optimized + unpriced_strategy: fallback targets: - virtual_key: together @@ -88,13 +89,15 @@ curl -s http://localhost:8080/v1/chat/completions \ }' ``` -The gateway estimates the input token cost across all targets with a compatible model, then routes the request to the cheapest provider. If cost data is unavailable for a target, it falls back to the first compatible target in the list. Check the `x-ferro-target` response header to see which provider was selected. +The gateway estimates prompt cost at roughly 4 characters/token — a routing heuristic, not a billing figure — across every target that serves the requested model, then routes to the cheapest priced one. `unpriced_strategy` decides what happens when a compatible target isn't in the catalog: `fallback` (shown above, and the default) prefers priced candidates and falls back to the first compatible unpriced one; `skip` refuses to route to an unpriced target at all (erroring if none is priced); `allow` treats an unpriced target as free, so it wins every draw. + +There is no response header naming which provider answered. To see it, add the `request-logger` plugin with `persist: true` (see [Configuration](/getting-started/configuration) for the request-log store env vars) and check the `provider` column, either via `GET /admin/logs` or directly against the `request_logs` table. --- ## 3. A/B test: compare GPT-4o vs Claude on 20% of traffic -Split live traffic between two providers. Every request is tagged with a variant label so you can aggregate quality metrics downstream. +Split live traffic between two providers by weight. Every request draws one variant — this is a real split, not shadow traffic — so quality and cost differences show up directly in your request logs. Register `openai` and `anthropic` by setting `OPENAI_API_KEY` and `ANTHROPIC_API_KEY`, then reference them by `virtual_key`. @@ -121,12 +124,23 @@ plugins: config: level: info persist: true - backend: postgres - dsn: postgres://ferro:ferro_secret@postgres:5432/ferro_logs?sslmode=disable + - name: request-logger + type: logging + stage: after_request + enabled: true + config: + level: info + persist: true + - name: request-logger + type: logging + stage: on_error + enabled: true + config: + level: info + persist: true ``` ```bash -# Send a request and check which variant was selected curl -s http://localhost:8080/v1/chat/completions \ -H "Content-Type: application/json" \ -H "Authorization: Bearer $API_KEY" \ @@ -138,18 +152,22 @@ curl -s http://localhost:8080/v1/chat/completions \ }' ``` -The variant `label` (`control` or `challenger`) is currently visible only in the gateway's DEBUG logs — it is not emitted to events or stored as a queryable log column. The request logger still persists latency and cost for every request, which you can aggregate in Postgres: +The `label` field (`control`/`challenger`) is a config-time identifier — it's used in validation error messages and to line up `target_key` with a variant — but it is not currently written to logs, metrics, or trace spans. (`ferro.routing.ab_variant_label` exists as a planned tracing attribute but isn't wired into emitted spans as of v1.4.1.) Because each variant here maps to a distinct provider, group by the `provider` column that `request-logger` persists to recover the split: ```sql SELECT - COUNT(*) AS requests, - AVG(latency_ms) AS avg_latency, - AVG(total_cost_usd) AS avg_cost + provider, + COUNT(*) AS requests, + AVG(duration_ms) AS avg_duration_ms, + AVG(ttft_ms) AS avg_ttft_ms, + AVG(cost_usd) AS avg_cost_usd FROM request_logs -WHERE created_at > NOW() - INTERVAL '24 hours'; +WHERE stage = 'after_request' + AND created_at > NOW() - INTERVAL '24 hours' +GROUP BY provider; ``` -To attribute results per variant, read the variant `label` from the gateway's DEBUG logs. +`cost_usd` is nullable — it's null when the catalog doesn't price the model, not zero — so `AVG(cost_usd)` silently drops those rows from the average rather than understating it. If your variants ever target the same provider (e.g. two models on the same backend), group by `model` instead, since `provider` alone won't distinguish them. --- @@ -200,16 +218,20 @@ curl -s http://localhost:8080/v1/chat/completions \ }' ``` -Content conditions are evaluated in order. The first match wins. If no condition matches, the request falls through to the first target in the `targets` list (OpenAI in this example). Regex patterns are compiled at startup; an invalid pattern causes a startup error. +Only user-role messages are inspected — content conditions are evaluated in order, and the first match wins. If no condition matches, the request goes to the first target in the `targets` list (OpenAI in this example). This is a named mode: once a rule matches, the gateway commits to that target and reports its failure rather than trying another. Regex patterns are Go RE2 syntax, compiled at startup — an invalid pattern causes a startup error. --- -## 5. Rate-limited free tier: 60 RPM per API key for your SaaS +## 5. Rate-limited free tier: 60 RPM and a $5 spend cap per API key -Expose the gateway as your SaaS AI endpoint. Each customer gets an API key with 60 requests per minute and a $5 spend cap. +Expose the gateway as your SaaS AI endpoint. Each customer gets an API key with 60 requests per minute and a $5 cumulative spend cap. Register `openai` by setting `OPENAI_API_KEY`, then reference it by `virtual_key`. +:::warning Multi-stage plugins need identical config +`budget` (like `response-cache` and `request-logger`) runs at two stages, and the gateway resolves both entries to one shared instance by comparing their config as JSON. If the `before_request` and `after_request` blocks don't match byte-for-byte, **the gateway refuses to start**. Copy the block below verbatim and only change `stage`. +::: + ```yaml title="config.yaml" strategy: mode: single @@ -228,7 +250,7 @@ plugins: key_rpm: 60 burst: 10 - # Spend cap: $5 per API key (check before request) + # Spend cap: $5 per API key (checked before every request) - name: budget type: guardrail stage: before_request @@ -240,15 +262,17 @@ plugins: output_per_m_tokens: 0.60 max_keys: 50000 - # Spend cap: record cost after response + # Same config, after_request — records cost from the response's token usage - name: budget type: guardrail stage: after_request enabled: true config: store_id: "free-tier" + spend_limit_usd: 5.0 input_per_m_tokens: 0.15 output_per_m_tokens: 0.60 + max_keys: 50000 ``` Normal request (succeeds): @@ -265,37 +289,222 @@ curl -s http://localhost:8080/v1/chat/completions \ }' ``` -When the rate limit is exceeded, the gateway returns a `429`: +When the rate limit is exceeded, the gateway returns `429`: ```json { "error": { - "message": "Rate limit exceeded: per-key limit (60 rpm)", + "message": "request rejected by rate-limit (before_request): per-key rate limit exceeded", "type": "rate_limit_error", "code": "rate_limit_exceeded" } } ``` -When the spend cap is hit, the gateway returns a `429` with a budget-specific message: +When the spend cap is hit, the gateway returns **`402 Payment Required`**, not `429` — a cumulative spend cap doesn't clear on a timer the way a rate limit does, so it uses a status no SDK's default retry policy touches: ```json { "error": { - "message": "Budget exceeded: spend limit of $5.00 USD reached for this API key", - "type": "budget_error", - "code": "budget_exceeded" + "message": "request rejected by budget (before_request): budget exceeded: spent $5.0000 of $5.00 limit", + "type": "insufficient_quota", + "code": "insufficient_quota" } } ``` -Rate checks execute in order: **global, then per-key, then per-user**. The budget plugin checks cumulative spend before forwarding the request and records the cost after the response. +Checks run in order — global rate limit, then per-key, then per-user — and the first denial wins. `budget` is a soft cap: it has no reservation step, so concurrent in-flight requests on one key can all pass the check and collectively overshoot by up to the in-flight count times one request's cost. Spend is in-memory and does not survive a restart. + +--- + +## 6. Self-hosted or preview models with `targets[].models` + +Route to a model your build's catalog and live discovery don't know about yet: a fine-tune behind your own OpenAI-compatible server, a preview id a provider hasn't published, or a regional deployment name. + +Register `openai` normally, then point it at your internal endpoint with `OPENAI_BASE_URL` and declare the extra model id on the target. + +```yaml title="config.yaml" +strategy: + mode: single + +targets: + - virtual_key: openai + models: + - my-org/llama-3.3-70b-ft-v2 +``` + +```bash +curl -s http://localhost:8080/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $API_KEY" \ + -d '{ + "model": "my-org/llama-3.3-70b-ft-v2", + "messages": [ + {"role": "user", "content": "Draft a release note for v2.4.0."} + ] + }' +``` + +`targets[].models` is purely additive — it never hides what the catalog or provider's own `/models` endpoint already reports, and declaring a model the catalog already knows about is a harmless no-op. Entries must be exact model ids; wildcards are rejected at load, because the routing index is an exact-match map. Declared models are fully routable and appear in `GET /v1/models`. Reach for this whenever a target serves a model no automatic source knows about — including a provider that exposes no `/models` endpoint to enumerate at all. + +--- + +## 7. Batch jobs and file uploads with `batch_target` + +`/v1/files*` and `/v1/batches*` carry no `model` field — they reference opaque, provider-scoped ids — so one `batch_target` serves the entire surface as a native pass-through with zero gateway routing state. + +```yaml title="config.yaml" +strategy: + mode: single + +targets: + - virtual_key: openai + +batch_target: openai +``` + +```bash +# Upload the batch input file +curl -s http://localhost:8080/v1/files \ + -H "Authorization: Bearer $API_KEY" \ + -F purpose="batch" \ + -F file="@requests.jsonl" + +# Create the batch job (input_file_id from the upload response above) +curl -s http://localhost:8080/v1/batches \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $API_KEY" \ + -d '{ + "input_file_id": "file-abc123", + "endpoint": "/v1/chat/completions", + "completion_window": "24h" + }' +``` + +Omit `batch_target` and both surfaces answer `501` instead of silently guessing a target. The named target's provider must support batch pass-through — currently `openai`, `azure-openai`, `groq`, `novita`, or `qwen` — and `batch_target` must name a target already listed under `targets`. + +--- + +## 8. Stateful conversations with `/v1/responses` + +`POST /v1/responses` routes by the body's `model` exactly like chat completions — full plugin pipeline, retry, circuit breaker — and is priced from the response's usage. The stateful id sub-routes (retrieve, delete, cancel, list input items) carry no model, so they pin to a single `responses_target`. + +```yaml title="config.yaml" +strategy: + mode: fallback + +targets: + - virtual_key: openai + - virtual_key: azure-openai + +responses_target: openai +``` + +```bash +# Create +curl -s http://localhost:8080/v1/responses \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $API_KEY" \ + -d '{ + "model": "gpt-4o", + "input": "Summarize the plot of Dune in two sentences." + }' + +# Retrieve (uses the id from the create response; served by responses_target) +curl -s http://localhost:8080/v1/responses/resp_abc123 \ + -H "Authorization: Bearer $API_KEY" +``` + +Omitting `responses_target` leaves `create` fully working — it routes by model like any other surface — while the id sub-routes answer `501`. `responses_target` must name a target already listed under `targets`. + +--- + +## 9. Native audio routing: speech-to-text and text-to-speech + +`/v1/audio/transcriptions`, `/v1/audio/translations`, and `/v1/audio/speech` are natively routed through the same targets, plugins, retry, and circuit-breaker pipeline as chat — not proxied pass-through. + +```yaml title="config.yaml" +strategy: + mode: fallback + +targets: + - virtual_key: groq + retry: + attempts: 2 + on_status_codes: [429, 502, 503] + - virtual_key: openai +``` + +Transcription (multipart upload, capped at 25 MiB): + +```bash +curl -s http://localhost:8080/v1/audio/transcriptions \ + -H "Authorization: Bearer $API_KEY" \ + -F file="@meeting.mp3" \ + -F model="whisper-large-v3" +``` + +Text-to-speech: + +```bash +curl -s http://localhost:8080/v1/audio/speech \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $API_KEY" \ + -d '{ + "model": "tts-1", + "input": "Your order has shipped.", + "voice": "alloy" + }' \ + --output speech.mp3 +``` + +Each target's registered provider must implement the surface you're calling: transcription/translation is served by `azure-openai`, `deepinfra`, `fireworks`, `groq`, `mistral`, `openai`, `sambanova`, and `together`; speech is served by `azure-openai`, `deepinfra`, `groq`, `mistral`, `openai`, and `together`. `fallback` here retries a failed transcription on the next target the same way it would a chat request. + +--- + +## 10. Zero-install agentic MCP over stdio + +Launch an MCP tool server directly from the gateway process with a package runner — no separate container, no `url` to stand up. This uses the `command` (stdio) transport instead of the `url` (HTTP) transport. + +Register `anthropic` by setting `ANTHROPIC_API_KEY`. + +```yaml title="config.yaml" +strategy: + mode: single + +targets: + - virtual_key: anthropic + +mcp_servers: + - name: brave-search + command: npx + args: + - -y + - "@modelcontextprotocol/server-brave-search" + env: + BRAVE_API_KEY: "${BRAVE_API_KEY}" + max_call_depth: 3 +``` + +```bash +curl -s http://localhost:8080/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $API_KEY" \ + -d '{ + "model": "claude-sonnet-4-6", + "messages": [ + {"role": "user", "content": "What is the latest stable version of PostgreSQL?"} + ] + }' +``` + +`npx -y` fetches the package on first launch and the subprocess runs for the gateway's lifetime. Critically, the subprocess does **not** inherit the gateway's environment — it gets only `PATH`, `HOME`, `LANG`, `TMPDIR`, and whatever you list under `env`, so `OPENAI_API_KEY` and `MASTER_KEY` never reach it. `env` (like `headers` on an HTTP MCP server) is the only credential channel for a stdio server, and `${VAR}` is resolved when the MCP client is constructed — set `BRAVE_API_KEY` in the gateway's own environment before starting it. --- -## 6. Agentic pipeline with filesystem MCP + Anthropic +## 11. Agentic pipeline with an HTTP filesystem MCP server + Anthropic -Connect a Model Context Protocol (MCP) tool server to the gateway. The gateway runs the full agentic tool-calling loop so your client receives a final text answer without implementing tool-calling logic. +Connect an MCP tool server over Streamable HTTP. The gateway runs the full agentic tool-calling loop so your client receives a final text answer without implementing tool-calling logic itself. Register `anthropic` by setting `ANTHROPIC_API_KEY`, then reference it by `virtual_key`. @@ -324,7 +533,7 @@ curl -s http://localhost:8080/v1/chat/completions \ -H "Content-Type: application/json" \ -H "Authorization: Bearer $API_KEY" \ -d '{ - "model": "claude-sonnet-4-20250514", + "model": "claude-sonnet-4-6", "messages": [ {"role": "user", "content": "Read the file /data/config.json and summarize what settings it contains."} ] @@ -333,27 +542,27 @@ curl -s http://localhost:8080/v1/chat/completions \ Behind the scenes the gateway: -1. Injects the available MCP tools (`read_file`, `list_directory`, `search_files`) into the chat completion request. +1. Injects the available MCP tools (`read_file`, `list_directory`, `search_files`) into the chat completion request — only when the request carries no `tools` of its own. 2. Receives a `tool_calls` response from Claude requesting `read_file` with path `/data/config.json`. 3. Executes the tool call against the MCP filesystem server. -4. Sends the tool result back to Claude. +4. Re-runs `before_request` guardrails, rate-limit, and budget checks for this loop turn, then sends the tool result back to Claude. 5. Returns Claude's final text summary to the client. -The entire agentic loop is transparent. The client sends a standard chat completion request and receives a standard text response. +The entire agentic loop is transparent to the caller — a standard chat completion request in, a standard text response out. To run the MCP filesystem server alongside the gateway in Docker Compose: ```yaml title="docker-compose.yml" services: gateway: - image: ghcr.io/ferrolabs/ferrogw:latest + image: ghcr.io/ferro-labs/ai-gateway:latest ports: - "8080:8080" environment: - GATEWAY_CONFIG: /etc/ferro/config.yaml + GATEWAY_CONFIG: /etc/ferrogw/config.yaml ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY} volumes: - - ./config.yaml:/etc/ferro/config.yaml:ro + - ./config.yaml:/etc/ferrogw/config.yaml:ro depends_on: - mcp-filesystem @@ -369,10 +578,57 @@ services: --- +## 12. Production hardening + +`GATEWAY_ENV=production` turns on startup safety checks that are off by default in development. + +```bash title="Environment" +export GATEWAY_ENV=production +export MASTER_KEY=fgw_your-master-key # generated by `ferrogw init`; the bootstrap admin credential +export CORS_ORIGINS=https://app.example.com,https://admin.example.com +export TRUSTED_PROXIES=10.0.0.0/8 # your load balancer's subnet +export API_KEY_STORE_BACKEND=postgres +export API_KEY_STORE_DSN=postgres://ferro:ferro_secret@postgres:5432/ferro_admin?sslmode=disable +``` + +```yaml title="docker-compose.yml" +services: + gateway: + image: ghcr.io/ferro-labs/ai-gateway:latest + restart: unless-stopped + ports: + - "8080:8080" + environment: + GATEWAY_ENV: production + MASTER_KEY: ${MASTER_KEY} + CORS_ORIGINS: ${CORS_ORIGINS} + TRUSTED_PROXIES: ${TRUSTED_PROXIES} + API_KEY_STORE_BACKEND: ${API_KEY_STORE_BACKEND} + API_KEY_STORE_DSN: ${API_KEY_STORE_DSN} + GATEWAY_CONFIG: /etc/ferrogw/config.yaml + OPENAI_API_KEY: ${OPENAI_API_KEY} + volumes: + - ./config.yaml:/etc/ferrogw/config.yaml:ro + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:8080/readyz"] + interval: 30s + timeout: 5s + retries: 3 +``` + +```bash +curl -s http://localhost:8080/readyz +``` + +`GATEWAY_ENV=production` **refuses to start** if `ALLOW_UNAUTHENTICATED_PROXY=true` or `CORS_ORIGINS` contains a literal `*` — `CORS_ORIGINS` is always matched literally against the request's `Origin` header, never as a wildcard pattern, so a bare `*` would allow no cross-origin request at all rather than every one. It only warns (doesn't refuse) on `RATE_LIMIT_RPS=0`, `ENABLE_PPROF=true`, and the default in-memory key store, since those are legitimate for some deployments. `MASTER_KEY` (generated by `ferrogw init`, always prefixed `fgw_`) is the bootstrap/break-glass admin credential — it has no key-store row and can't be revoked without a restart, so issue day-to-day operator keys via `POST /admin/keys` and reserve `MASTER_KEY` for emergencies. `TRUSTED_PROXIES` lists the CIDRs whose `X-Forwarded-For`/`X-Real-IP` are honored for client-IP resolution (used by the per-IP rate limiter); it defaults to loopback only, so behind a real load balancer every request looks like it came from the LB until this is set. Health-check `/readyz`, not `/health` — `/health` returns `200` even when every configured target is unroutable, which is exactly the outage a health check should catch. + +--- + ## Related pages -- [Routing policies](/guides/routing-policies) -- [Plugins](/guides/plugins) +- [Routing strategies](/routing) +- [Plugins](/plugins) - [MCP integration](/guides/mcp) +- [Authentication](/guides/auth) - [Benchmarks](/benchmarks) - [Configuration](/getting-started/configuration) diff --git a/docs/guides/virtual-keys.mdx b/docs/guides/virtual-keys.mdx index 64e2d62..092ff1a 100644 --- a/docs/guides/virtual-keys.mdx +++ b/docs/guides/virtual-keys.mdx @@ -1,7 +1,7 @@ --- title: Virtual Keys vs API Keys -description: The two distinct key concepts in the Ferro Labs AI Gateway — the config virtual_key that names a provider credential set, and fgw_ admin API keys for client auth. -keywords: [virtual key, fgw api key, gateway target, provider credentials, bearer token, admin keys] +description: virtual_key names a provider credential set in config; fgw_ admin API keys are client bearer credentials for gateway auth. How they differ, where each lives. +keywords: [virtual key, fgw api key, gateway target, provider credentials, bearer token, admin keys, dashboard session] --- The word "key" shows up in two completely different places in the Ferro Labs AI @@ -67,11 +67,30 @@ and is safe to commit. The provider's real API key is resolved separately from the environment at request time. ::: +A target can also declare `models[]` alongside its `virtual_key` — model IDs +the operator asserts that target serves, additive to whatever the model +catalog and live discovery already report. It's the provider-agnostic way to +route a model a provider doesn't otherwise advertise (a preview ID, a +self-hosted deployment, a provider with no `/models` endpoint): + +```yaml +targets: + - virtual_key: gemini + models: + - gemini-2.5-flash # joins the routing index and /v1/models +``` + +See [Configuration](/getting-started/configuration) for the full `targets[]` +schema. + ## Admin API keys (`fgw_...`) — client credentials Admin API keys are the credentials **clients use to authenticate to the gateway**. They are minted via `POST /admin/keys` and returned as a string with -an `fgw_` prefix (32 random bytes, hex-encoded — see `internal/admin/keys.go`). +an `fgw_` prefix. The admin control plane that issues and validates them is +split across three packages under `internal/admin/` — `model` (the `APIKey` +type and scope rules), `repository` (the key store), and `handlers` (the HTTP +surface) — there's no single `keys.go` file to point to. A client then presents the key as a Bearer token on inference routes: @@ -84,19 +103,42 @@ curl http://localhost:8080/v1/chat/completions \ Each key is a record with a lifecycle: -- **Scopes** — capabilities granted to the key. When none are supplied at - creation, the key defaults to the admin scope. -- **Expiry** — an optional `expires_at`. `ValidateKey` rejects a key once the - expiry has passed. -- **Rotation** — `RotateKey` swaps the key string in place (stamping - `rotated_at`) while keeping the same ID, scopes, and usage history. -- **Revocation** — `Revoke` marks a key inactive so it stops validating, without - deleting its record. +- **Scopes** — `admin` or `read_only`. When none are supplied at creation, the + key defaults to **`read_only`** — least privilege, not admin. +- **Expiry** — an optional `expires_at`. A key past its expiry stops + authenticating. `PUT /admin/keys/{id}` can extend it by sending a new + `expires_at`, or remove it entirely with `"clear_expiration": true`. +- **Rotation** — `POST /admin/keys/{id}/rotate` swaps the key string in place + (stamping `rotated_at`) while keeping the same ID, scopes, and usage history. +- **Revocation vs. deletion** — these are not the same operation. `POST + /admin/keys/{id}/revoke` marks the key inactive so it immediately stops + validating, but keeps the record (and its usage history) in `GET + /admin/keys` and in request-log attribution. `DELETE /admin/keys/{id}` + removes the record outright — it's irreversible, and existing request-log + rows keep the now-orphaned `api_key_id` but can no longer resolve it to a + name. Prefer revoke when you want the trail; delete when you don't need it. - **Usage** — every successful validation bumps `usage_count` and `last_used_at`. +A key can't delete, revoke, or de-scope itself, and the last remaining `admin` +key can't be deleted, revoked, or stripped of the `admin` scope — both are +refused with `409` to prevent a self-inflicted lockout. + When listed, the stored key string is masked to its first 8 characters (`fgw_...`) so the full secret is never echoed back by the admin API. +### Dashboard sessions are not API keys + +`POST /admin/session` exchanges an API key (or `MASTER_KEY`) for a short-lived +dashboard **session** token, prefixed `fgws_` — deliberately distinct from an +API key's `fgw_` prefix so the two can never be confused. A session is a +separate record from the `APIKey` it was minted from: it carries its own ID, +inherits the source key's scopes at mint time, and expires on its own clock +(24h absolute, or 1h after the last request — whichever comes first). Signing +out (`DELETE /admin/session`) or an admin revoking one (`DELETE +/admin/sessions/{id}`, or all at once with `DELETE /admin/sessions`) deletes +the session row rather than marking it, so it stops validating immediately. +Revoking a session never touches the API key it came from. + ## Side-by-side | | `virtual_key` (config) | Admin API key (`fgw_...`) | @@ -115,5 +157,5 @@ When listed, the stored key string is masked to its first 8 characters admin API keys. - [Configuration](/getting-started/configuration) — full `targets[]` and strategy reference. -- [Providers configuration](/guides/providers-config) — which env var each +- [Providers configuration](/providers/configuration) — which env var each provider's `virtual_key` resolves to. diff --git a/docs/guides/why-ferro.mdx b/docs/guides/why-ferro.mdx index bd47ee6..3b02925 100644 --- a/docs/guides/why-ferro.mdx +++ b/docs/guides/why-ferro.mdx @@ -1,6 +1,7 @@ --- -title: Why Ferro Labs AI Gateway -description: "Compare Ferro Labs AI Gateway to LiteLLM, Portkey, Bifrost, and Cloudflare. An honest guide to choosing the right open-source AI gateway." +title: "LiteLLM & Portkey Alternatives — an Honest AI Gateway Comparison" +sidebar_label: Comparisons & Alternatives +description: "Looking for a LiteLLM alternative or self-hosted Portkey replacement? An honest comparison of Ferro Labs AI Gateway vs LiteLLM, Portkey, Bifrost, and Cloudflare." keywords: - LiteLLM alternative - Portkey alternative @@ -9,9 +10,7 @@ keywords: - open source AI gateway Go --- -# Why Ferro Labs AI Gateway - -AI gateways differ more than they look. Here's how to think about the choice. +Looking for a **LiteLLM alternative**, a self-hosted **Portkey replacement**, or just an honest comparison before picking an AI gateway? AI gateways differ more than they look — this page compares Ferro Labs AI Gateway with LiteLLM, Portkey, Bifrost, and Cloudflare AI Gateway on the criteria that actually decide the choice, and tells you when each one is the better fit (including when it isn't us). ## Decision Table @@ -55,7 +54,7 @@ When you need multi-tenant isolation, a dashboard, enterprise plugins, and someo These are concrete differences, not marketing claims. -### 1. Zero runtime dependencies +### 1. Single static binary Ferro Labs AI Gateway compiles to a single Go binary. The Docker image is under 20 MB. There is no interpreter, no virtual environment, no package manager involved at deploy time. This matters when you are running in constrained environments or want reproducible deploys. @@ -83,8 +82,11 @@ Ferro Labs OSS is Apache 2.0 licensed. There are no "community" vs. "enterprise" Bifrost is MIT-licensed, which is also permissive. LiteLLM is Apache 2.0 as well. Portkey and Cloudflare AI Gateway are proprietary. Pick the license model that matches your organization's requirements. ::: -## Related pages +## Ready to switch? + +Each migration is a base-URL swap plus a config translation — the guides walk through both: - [Migrating from LiteLLM](/guides/migration-litellm) - [Migrating from Portkey](/guides/migration-portkey) -- [Benchmarks](/benchmarks) +- [Migrating from OpenRouter](/guides/migration-openrouter) +- [Benchmarks](/benchmarks) — the latency numbers behind the table above diff --git a/docs/integrations/deployment/fly-io.mdx b/docs/integrations/deployment/fly-io.mdx index 1628246..ac8b0f4 100644 --- a/docs/integrations/deployment/fly-io.mdx +++ b/docs/integrations/deployment/fly-io.mdx @@ -8,24 +8,6 @@ keywords: - Fly.io LLM gateway --- -import Head from '@docusaurus/Head'; - - - - - # Deploy to Fly.io Fly.io runs Docker containers on bare-metal servers worldwide with sub-second boot times. It is one of the fastest ways to get a self-hosted AI Gateway into production without managing infrastructure. diff --git a/docs/integrations/deployment/kubernetes.mdx b/docs/integrations/deployment/kubernetes.mdx index d6fff26..95b8d29 100644 --- a/docs/integrations/deployment/kubernetes.mdx +++ b/docs/integrations/deployment/kubernetes.mdx @@ -1,6 +1,6 @@ --- title: Kubernetes deployment -description: Deploy the Ferro Labs AI Gateway to Kubernetes — Deployment, Service, ConfigMap, Secret, HPA, and readiness/liveness probe manifests with a production-ready values example. +description: "Deploy the Ferro Labs AI Gateway to Kubernetes — Deployment, Service, ConfigMap, Secret, HPA, and readiness/liveness probe manifests, production-ready." keywords: [Kubernetes AI gateway, deploy LLM gateway Kubernetes, Kubernetes LLM proxy, AI gateway K8s, Helm values AI gateway] --- diff --git a/docs/integrations/overview.mdx b/docs/integrations/overview.mdx index 4ac33fc..e753863 100644 --- a/docs/integrations/overview.mdx +++ b/docs/integrations/overview.mdx @@ -71,5 +71,5 @@ One-click deploys and production-ready configurations for every major platform: The gateway supports **30 AI providers** out of the box, including OpenAI, Anthropic, Google, Mistral, Cohere, and more. No vendor lock-in — switch between providers by changing a single route configuration. -- [Providers overview](/guides/providers) — Full list of supported providers -- [Provider configuration](/guides/providers-config) — Environment variables and setup +- [Providers overview](/providers) — Full list of supported providers +- [Provider configuration](/providers/configuration) — Environment variables and setup diff --git a/docs/integrations/sdk/python/reference.mdx b/docs/integrations/sdk/python/reference.mdx index f7e0c28..c282e00 100644 --- a/docs/integrations/sdk/python/reference.mdx +++ b/docs/integrations/sdk/python/reference.mdx @@ -10,7 +10,7 @@ Complete API reference for the `ferrolabsai` Python SDK. ## FerroClient -The synchronous client for interacting with the Ferro AI Gateway. `AsyncFerroClient` has an identical interface but returns coroutines. +The synchronous client for interacting with the Ferro Labs AI Gateway. `AsyncFerroClient` has an identical interface but returns coroutines. ```python from ferrolabsai import FerroClient diff --git a/docs/intro.mdx b/docs/intro.mdx index 23c575a..ef8b7bd 100644 --- a/docs/intro.mdx +++ b/docs/intro.mdx @@ -1,29 +1,24 @@ --- -title: Introduction -description: "Ferro Labs AI Gateway is an open-source Go LLM proxy that routes traffic across 30 AI providers and 2,500+ models with sub-millisecond overhead. Drop-in OpenAI-compatible API with 8 routing strategies, 6 built-in plugins, and MCP tool-calling." -keywords: [AI gateway, open source AI gateway, LLM proxy, OpenAI compatible, Go LLM proxy, self-hosted LLM gateway, AI middleware, multi-provider AI, LLM routing, AI gateway v1.1.0] +title: Ferro Labs AI Gateway +description: "Open-source, self-hosted AI gateway in Go: 30 providers and 2,500+ models behind one OpenAI-compatible API — routing strategies, plugins, MCP, embedded dashboard." +keywords: [AI gateway, open source AI gateway, LLM proxy, OpenAI compatible, Go LLM proxy, self-hosted LLM gateway, AI middleware, multi-provider AI, LLM routing, AI gateway v1.4] +slug: / +hide_table_of_contents: false --- -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; - # Ferro Labs AI Gateway -You're calling OpenAI directly. When they go down, your product goes down. When they raise prices, you scramble. When you need to test a new model, you rewrite integration code. When you want observability, you build it yourself. +**Ferro Labs AI Gateway** is an **open-source, self-hosted AI gateway**: a single Go binary that sits in front of all your LLM traffic, speaks an OpenAI-compatible API, and routes requests across **30 providers and 2,500+ models** — with safety and cost policies enforced by built-in plugins, production observability, and a React **dashboard embedded in the binary**, served at the gateway's own root. No client code changes required. -**Ferro Labs AI Gateway** is a single Go binary that sits in front of all your LLM traffic. It exposes an OpenAI-compatible API, routes requests across 30 providers and 2,500+ models, enforces safety policies with 6 built-in plugins, and emits production observability — without changing your existing client code. +Why put a gateway in front at all? Because you're calling one LLM provider directly: when they go down, your product goes down. When they raise prices, you scramble. When you want to try a new model, you rewrite integration code. When you need observability, you build it yourself. -:::tip Ferro Labs Managed — Managed AI Gateway -Ferro Labs Managed wraps the open-source gateway with multi-tenancy, a dashboard, durable billing, semantic caching, and 5 enterprise security plugins. [Join the early access waitlist →](https://www.ferrolabs.ai/) -::: +Built for production from the start: sub-millisecond p99 routing overhead ([benchmarks](/benchmarks)), retries and circuit breakers under [every routing strategy](/routing), per-target concurrency limits, and `/livez` + `/readyz` probes for orchestrators. :::tip Ready to jump in? -[Quickstart — up and running in 30 seconds →](/getting-started/quickstart) +[Quickstart — up and running in a couple of minutes →](/getting-started/quickstart) ::: -## What is it? - -Drop the gateway in front of your LLM traffic. Set `base_url` to the gateway endpoint. That's it — your OpenAI SDK, LangChain, LlamaIndex, or curl commands continue to work unchanged. +## Install in one command ```bash export OPENAI_API_KEY=sk-... @@ -32,92 +27,61 @@ export ANTHROPIC_API_KEY=sk-ant-... docker run -d -p 8080:8080 \ -e OPENAI_API_KEY \ -e ANTHROPIC_API_KEY \ + -e MASTER_KEY=$(openssl rand -hex 24) \ ghcr.io/ferro-labs/ai-gateway:latest ``` -Then send requests to `http://localhost:8080/v1/chat/completions` exactly as you would to OpenAI. +Send requests to `http://localhost:8080/v1/chat/completions` exactly as you would to OpenAI, authenticating with the `MASTER_KEY` you set (or a scoped key you issue from it). Open **`http://localhost:8080/`** in a browser for the built-in dashboard. New here? The [quickstart](/getting-started/quickstart) walks through `ferrogw init`, which scaffolds a config and a key for you. + +:::note `/v1/*` requires a key by default +Every inference route needs a bearer token unless you explicitly set `ALLOW_UNAUTHENTICATED_PROXY=true`. See [Authentication](/guides/auth). +::: ## Key capabilities | Capability | Details | |---|---| -| **30 AI providers** | OpenAI, Anthropic, Gemini, Mistral, Groq, Cohere, DeepSeek, Together, Perplexity, Fireworks, AI21, Azure OpenAI, Azure Foundry, xAI, Ollama, Ollama Cloud, Replicate, AWS Bedrock, Vertex AI, Hugging Face, Cerebras, NVIDIA NIM, Cloudflare Workers AI, Databricks, Novita AI, Qwen, Moonshot AI, SambaNova, DeepInfra, OpenRouter | -| **8 routing strategies** | Single, Fallback, Weighted, Conditional, Least-Latency, Cost-Optimized, Content-Based, A/B Test | -| **6 OSS + 5 Ferro Labs Managed plugins** | Word filter, max-token, response cache, request logger, rate limit, budget (OSS) — plus PII redact, secret scan, prompt shield, schema guard, regex guard ([Ferro Labs Managed](https://www.ferrolabs.ai/)) | -| **MCP integration** | Agentic tool-calling loop via Model Context Protocol servers with streaming support (v1.0.0) | -| **Observability** | Prometheus metrics, structured JSON logs with trace IDs, deep `/health` per provider | -| **Resiliency** | Per-target circuit breakers, retry with exponential backoff, per-status-code retry config | -| **OpenAI compatible** | Chat completions, embeddings, images, and model listing — same wire format | -| **Built in Go** | Single binary, zero runtime dependencies, sub-millisecond p99 overhead at 500 RPS. [See benchmarks →](/benchmarks) | - -:::info Why Go? -Go's goroutine scheduler, low-GC overhead, and lack of a GIL make it ideal for a latency-sensitive proxy. The gateway ships as a single static binary under 20MB with zero runtime dependencies. [Published benchmarks](/benchmarks) show sub-millisecond p99 overhead at 500 RPS — while Python-based gateways degrade beyond 200 RPS. +| **30 AI providers** | OpenAI, Anthropic, Gemini, Vertex AI, Mistral, Groq, Cohere, DeepSeek, Together, Perplexity, Fireworks, AI21, Azure OpenAI, Azure AI Foundry, xAI, Ollama, Ollama Cloud, Replicate, AWS Bedrock, Hugging Face, Cerebras, NVIDIA NIM, Cloudflare Workers AI, Databricks, Novita, Qwen, Moonshot, SambaNova, DeepInfra, OpenRouter — [see the matrix →](/providers) | +| **8 routing strategies** | Single, fallback, weighted load balancing, least-latency, cost-optimized, conditional, content-based, and A/B test — [routing guide →](/routing) | +| **6 built-in plugins** | Word filter, max-token, rate limit, budget, response cache, request logger — plus [5 Ferro Labs Managed](/plugins/enterprise) security plugins. [Plugin catalogue →](/plugins) | +| **Unified request pipeline** | Chat, streaming, embeddings, and image generation share one routing pipeline — retry, circuit breaking, per-target concurrency, timeouts, metrics, and request logging are true of every surface. | +| **Native endpoints** | Chat completions, embeddings, images, **rerank, moderations, audio (STT/TTS), files & batches, and `/v1/responses`** — governed and priced, not blindly proxied. [API reference →](/api-reference/overview) | +| **MCP tool-calling** | Model Context Protocol servers over **HTTP or stdio** (run any `npx`/`uvx` server as a managed subprocess), with per-turn guardrails and budgeting on agentic loops. [MCP guide →](/guides/mcp) | +| **Embedded dashboard** | Overview, keys, request logs, analytics, config history, playground, audit, and tracing — compiled into the binary, served at the root. [Dashboard tour →](/guides/dashboard) | +| **Observability** | Prometheus `/metrics` (scoped), OpenTelemetry tracing, structured JSON logs with 32-hex trace IDs, and `/livez` + `/readyz` probes. [Observability →](/guides/observability) | +| **Built in Go** | Ships as a single static binary; sub-millisecond p99 routing overhead. [See benchmarks →](/benchmarks) | + +:::tip What's new in v1.4 +The dashboard now ships **embedded** in the OSS binary, all four core surfaces share **one routing pipeline** (with retry honoured under every strategy), and rerank / moderations / audio / responses became first-class routed endpoints. Some breaking changes ship with it — read the [changelog](/changelog) before upgrading. +::: + +## Where to next + +### Get started +- [Overview](/getting-started/overview) — when and why to use the gateway +- [Quickstart](/getting-started/quickstart) — Docker, `ferrogw init`, your first request +- [Architecture](/getting-started/architecture) — components and the unified request pipeline +- [Configuration](/getting-started/configuration) — the full config reference +- [Concepts](/getting-started/concepts) — routing, plugins, observability, MCP + +### Core reference +- [Providers](/providers) — all 30 providers, supported endpoints, and how to enable them +- [Routing](/routing) — the 8 strategies, with pool-vs-named behaviour and examples +- [Plugins](/plugins) — the 6 built-in plugins, config, and pipeline stages +- [Dashboard](/guides/dashboard) — the embedded console +- [API reference](/api-reference/overview) — endpoints, streaming, admin, and errors + +### Operate +- [Authentication](/guides/auth) — master key, scoped keys, and sessions +- [Observability](/guides/observability) & [Monitoring](/operations/monitoring) +- [Server settings](/operations/server-settings) — every environment variable +- [Troubleshooting](/guides/troubleshooting) + +:::info Ferro Labs Managed +Need multi-tenancy, hosted infrastructure, semantic caching, and enterprise security plugins on top of the OSS engine? [Ferro Labs Managed](/ferrocloud/overview) is in early access — [join the waitlist →](https://www.ferrolabs.ai/). ::: -## Docs map - -### Getting started -- [Overview](/getting-started/overview) — When and why to use the gateway -- [Architecture](/getting-started/architecture) — Component diagrams and data flow -- [Request lifecycle](/getting-started/request-lifecycle) — Step-by-step request flow -- [Quickstart](/getting-started/quickstart) — Docker, build from source, first request -- [Concepts](/getting-started/concepts) — Core ideas: routing, plugins, observability, MCP -- [Configuration](/getting-started/configuration) — Full config reference - -### Guides -- [Providers](/guides/providers) — All 30 providers and supported capabilities -- [Provider configuration](/guides/providers-config) — Environment variables per provider -- [Authentication](/guides/auth) — API key configuration -- [Routing policies](/guides/routing-policies) — All 8 routing strategies with examples -- [Plugins](/guides/plugins) — 6 built-in + 5 managed plugins with YAML config -- [MCP integration](/guides/mcp) — Model Context Protocol tool servers -- [Observability](/guides/observability) — Metrics, logs, health checks -- [Rate limiting](/guides/rate-limiting) — IP-level and request-level limiting -- [Admin auth](/guides/admin-auth) — Admin API scopes and tokens -- [Use cases](/guides/use-cases) — Recipe-style configurations for common scenarios -- [Why Ferro Labs](/guides/why-ferro) — Comparison with LiteLLM, Portkey, and other gateways - -### Performance -- [Benchmarks](/benchmarks) — Published Go vs Python gateway performance data - -### Integrations -- [Integrations overview](/integrations/overview) — SDKs, frameworks, deployment, and providers -- [Python SDK quickstart](/integrations/sdk/python/quickstart) — Install `ferrolabsai` and send your first request -- [Python SDK reference](/integrations/sdk/python/reference) — Full API reference -- [Go SDK](/integrations/sdk/go) — Embed the gateway, write custom plugins -- [OpenAI-compatible SDKs](/integrations/sdk/openai-compatible) — Use any OpenAI SDK with zero changes - -### Deployment -- [Railway](/integrations/deployment/railway) — One-click Railway deploy (SQLite or PostgreSQL) -- [Render](/integrations/deployment/render) — One-click Render deploy with managed PostgreSQL -- [Docker Compose](/integrations/deployment/docker-compose) — Production-like deployment -- [Kubernetes](/integrations/deployment/kubernetes) — Helm chart and manifests -- [Fly.io](/integrations/deployment/fly-io) — Deploy to Fly.io - -### Operations & reference -- [Monitoring](/operations/monitoring) — Prometheus queries, alerting, dashboards -- [Request logging](/operations/request-logging) — Persistent log backends -- [Server settings](/operations/server-settings) — All environment variables -- [API reference](/api-reference/overview) — Endpoints, request format, admin API -- [Security](/security/data-handling) — Data handling and least-privilege configuration -- [Troubleshooting](/guides/troubleshooting) — Common issues and fixes -- [FAQ](/faq) — Common questions - -### Ferro Labs Managed (Managed) -- [Ferro Labs Managed overview](/ferrocloud/overview) — Managed multi-tenant AI gateway -- [Semantic caching](/ferrocloud/semantic-cache) — pgvector-based semantic response cache -- [OSS vs Ferro Labs Managed](/guides/oss-vs-ferrocloud) — Feature comparison - - +{/* No page-local JSON-LD here: the site-wide graph in docusaurus.config.ts + already declares the one canonical SoftwareApplication entity. A second, + un-@id'd copy with a different url/description read as a conflicting + duplicate entity to consumers. */} diff --git a/docs/operations/cli-reference.mdx b/docs/operations/cli-reference.mdx index 93b92c0..1ce9338 100644 --- a/docs/operations/cli-reference.mdx +++ b/docs/operations/cli-reference.mdx @@ -1,6 +1,6 @@ --- title: ferrogw CLI reference -description: Command-line reference for the ferrogw binary — init, serve, validate, doctor, status, version, plugins, and admin subcommands, with global flags and env vars. +description: "Reference for the ferrogw CLI: init, serve, validate, doctor, status, plugins, version, and admin subcommands, with every global flag and environment variable." keywords: [ferrogw CLI, AI gateway CLI, ferrogw commands, ferrogw admin, gateway command line, LLM gateway CLI reference] --- @@ -61,29 +61,34 @@ ferrogw init [flags] **Example** ```bash +export OPENAI_API_KEY=sk-... ANTHROPIC_API_KEY=sk-ant-... ferrogw init ``` ```text Ferro Labs AI Gateway -- Setup - ✔ Created config.yaml - ✔ Master key: fgw_4f9c2a1b7e3d8650a1c4f0b29d6e7a83 + [OK] Created config.yaml + [OK] Targets from detected credentials: openai, anthropic + [OK] Master key: fgw_4f9c2a1b7e3d8650a1c4f0b29d6e7a83 - ⚠ Save this key -- you need it for the dashboard and API. + [!] Save this key -- you need it for the Admin API and web application. export MASTER_KEY=fgw_4f9c2a1b7e3d8650a1c4f0b29d6e7a83 Next steps: - 1. Set provider API keys (e.g. export OPENAI_API_KEY=sk-...) - 2. Start the gateway: ferrogw serve - 3. Open dashboard: http://localhost:8080/dashboard + 1. Use this config: export GATEWAY_CONFIG=config.yaml + 2. Set provider API keys (e.g. export OPENAI_API_KEY=sk-...) + 3. Start the gateway: ferrogw serve + 4. Check readiness: curl http://localhost:8080/readyz ``` +The dashboard is served from the gateway's own root once it's running — there's no separate `/dashboard` route or standalone web container to open. + :::warning -The master key is shown **once**. Store it securely (export it as `MASTER_KEY` or place it in your secret manager). If an existing config file is present, `init` skips writing it but still prints a freshly generated key. +The master key is shown **once** and is never written to disk — store it securely (export it as `MASTER_KEY` or place it in your secret manager). If an existing config file is present, `init` leaves that file alone and generates **no** new master key: the key already in use for that deployment keeps working, and reprinting one would just be a credential that looks live and authenticates against nothing. ::: -The generated `config.yaml` starts with a `fallback` strategy across `openai` and `anthropic` targets, with other providers commented out for you to enable. +`init` always uses a `fallback` strategy, but its `targets` are scaffolded from whatever provider credentials the environment already holds — the same detection `ferrogw serve` uses to auto-register providers — so the file it writes is one this deployment can actually serve on the first request. Each detected provider (`OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, and so on) becomes a target, in provider-registry order. If no provider credentials are found at all, `init` writes a single placeholder target (`openai`) with a comment explaining that a real key still needs to be set before the gateway can route anything. ### `ferrogw serve` @@ -108,7 +113,12 @@ The listen port comes from `PORT` (default `8080`), and the config path from `GA ### `ferrogw validate` -Loads and validates a configuration file **offline** — no running gateway is required. Accepts YAML or JSON (auto-detected). Exits non-zero if the file fails to load or fails validation. +Loads and validates a configuration file **offline** — no running gateway is required. Accepts YAML or JSON (auto-detected). Validation runs two layers, both of which `ferrogw serve` runs too: + +- **Config-level** — target/strategy shape, alias chains, MCP server entries, and (since v1.4.0) that any plugin listed at multiple stages — `response-cache`, `budget`, and `request-logger` all normally are — carries byte-identical config at each stage it's listed at. +- **Binary-level** — every target's `virtual_key` must name a provider this binary registers, and every enabled plugin's `name` and `stage` must resolve against what this binary ships. A disabled plugin entry is not checked, matching what `serve` skips too. + +It does not resolve `${VAR}` references, contact any provider, or check that credentials exist — a target naming a real provider whose API key is only set in production is a valid config here. Exits non-zero if the file fails to load or fails either layer. **Usage** @@ -123,7 +133,7 @@ ferrogw validate config.yaml ``` ```text -✔ Config is valid +[OK] Config is valid Strategy: fallback Targets: 2 Providers: openai, anthropic @@ -136,10 +146,14 @@ With `--format json` or `--format yaml`, the parsed config is printed in that fo Runs a set of environment, configuration, and connectivity checks to help diagnose setup problems. It reports: - **Provider API keys** — presence of `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GEMINI_API_KEY`, `GROQ_API_KEY`, `MISTRAL_API_KEY`. -- **Configuration** — loads and validates the file at `GATEWAY_CONFIG` (if set). +- **Configuration** — if `GATEWAY_CONFIG` is set, loads the file and runs the same two-layer check `ferrogw validate` does: config-level validation (including the multi-stage plugin agreement rule) plus provider-id and plugin name/stage resolution against this binary. An invalid config is reported in red here too, matching what `validate` and `serve` would reject. - **Auth** — whether `MASTER_KEY` is set. - **Gateway connectivity** — a `GET /health` round-trip to the configured gateway URL, with latency. +:::note +`doctor` only exits non-zero when the loaded config is invalid. A missing provider key or an unreachable gateway are reported as findings, not treated as failures — that's what makes `ferrogw doctor` safe to run against a laptop with no gateway running yet, while still catching a config a pipeline would need to fail on. +::: + **Usage** ```bash @@ -154,27 +168,33 @@ ferrogw doctor ```text Provider API Keys - ✔ openai - ✔ anthropic - - gemini - - groq - - mistral + [OK] openai + [OK] anthropic + [-] gemini + [-] groq + [-] mistral 2 found Configuration - ✔ config.yaml (strategy=fallback, targets=2) + [OK] config.yaml (strategy=fallback, targets=2) Auth - ✔ MASTER_KEY is set + [OK] MASTER_KEY is set Gateway Connectivity - ✔ http://localhost:8080 -- healthy (3ms) + [OK] http://localhost:8080 -- healthy (3ms) ``` ### `ferrogw status` -Checks the health of a running gateway by calling `GET /health`, then reports the version and provider/model counts (when available from `/admin/providers`). +Checks the health of a running gateway by calling `GET /health`, then reports the version and provider/model counts (when available from `GET /admin/providers`). + +- **Exits non-zero when the gateway is unreachable.** The connection or timeout error is returned, not swallowed, so `ferrogw status || alert` can see it. A gateway that *answers* — including a `503`-degraded `/health` — is still reachable and exits `0`; the degraded status is printed as a finding, not treated as a failure. +- **Diagnostics go to stderr.** The "gateway unreachable" error included, keeping stdout the machine-readable channel so `ferrogw status | jq` doesn't have to sort output from errors. +- **Colour is suppressed off a terminal.** Piping or redirecting output (or setting `NO_COLOR`) drops the ANSI codes, so a log file or `grep` sees plain ASCII text. +- **`--format json`/`--format yaml` are refused.** `status` reports a human-readable narrative, not structured data — there's no useful machine encoding for it, so asking for one is a command error rather than silently getting ANSI-decorated text back. +- **Takes no positional arguments.** A stray one is rejected rather than silently ignored. **Usage** @@ -189,8 +209,8 @@ ferrogw status --gateway-url http://localhost:8080 ``` ```text - ✔ http://localhost:8080 -- healthy (4ms) - Version: 1.1.0 + [OK] http://localhost:8080 -- healthy (4ms) + Version: 1.4.1 Providers: 30 (412 models) ``` @@ -237,7 +257,7 @@ ferrogw version ``` ```text - Version 1.1.0 + Version 1.4.1 Commit a1b2c3d Built 2026-06-01T12:00:00Z Go go1.25.0 @@ -272,6 +292,10 @@ The `admin` command groups operations into four areas: `keys`, `config`, `logs`, ferrogw admin keys create --name ci-bot --scope read_only --expires-in 720h ``` +:::note +`admin keys create` and `admin keys rotate` fail rather than print `null` when the Admin API answers `2xx` with an empty body — a payload was requested and none arrived, so that's reported as an error, not a silently empty result. `admin keys revoke` expects `204 No Content`; anything else, including a body-less redirect, is reported as a failure instead of being read as success. +::: + #### `admin config` — runtime configuration | Command | Description | @@ -331,6 +355,10 @@ These persistent flags apply to all subcommands. The API key resolves in order: `--api-key` flag → `FERROGW_API_KEY` → `MASTER_KEY`. This means once you export `MASTER_KEY`, the admin commands authenticate without any extra flag. ::: +:::note +`--format` only changes output for the commands that return structured data — `validate`, `plugins`, `version`, and the `admin` subcommands. `init`, `doctor`, and `status` render a human-readable report only and refuse any `--format` other than `table`. +::: + ## Environment variables `ferrogw` reads the following environment variables. Server-side variables are read by `serve`; CLI variables are read by the Admin API client. @@ -338,11 +366,19 @@ The API key resolves in order: `--api-key` flag → `FERROGW_API_KEY` → `MASTE | Variable | Read by | Description | |---|---|---| | `GATEWAY_CONFIG` | `serve`, `doctor` | Path to the config file (YAML or JSON) | +| `GATEWAY_ENV` | `serve` | Set to `production` (case-insensitive) to turn on production safety checks — refuses `ALLOW_UNAUTHENTICATED_PROXY=true` and a `*` in `CORS_ORIGINS`; warns on `RATE_LIMIT_RPS=0`, `ENABLE_PPROF`, and the in-memory key store | | `PORT` | `serve` | HTTP listen port (default `8080`) | -| `MASTER_KEY` | `serve`, admin CLI | Admin + proxy credential; generated by `ferrogw init` | +| `MASTER_KEY` | `serve`, admin CLI | Bootstrap/break-glass admin credential; generated by `ferrogw init` | | `FERROGW_URL` | CLI | Gateway base URL when `--gateway-url` is not set | | `FERROGW_API_KEY` | CLI | Admin API key when `--api-key` is not set | -| `ALLOW_UNAUTHENTICATED_PROXY` | `serve` | Set to `true` to leave proxy routes unauthenticated (development only — not recommended for production) | +| `ALLOW_UNAUTHENTICATED_PROXY` | `serve` | Set to `true` to leave proxy routes unauthenticated (development only — refused when `GATEWAY_ENV=production`) | | `ENABLE_PPROF` | `serve` | Set to `true` to mount `/debug/pprof/*` profiling routes | -Provider credentials (`OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, and the rest) are documented in the [Providers guide](/guides/providers). For the full server configuration schema, see [Configuration](/getting-started/configuration). +Provider credentials (`OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, and the rest) are documented in [Provider configuration](/providers/configuration). For the full server configuration schema, see [Configuration](/getting-started/configuration). + +## Related + +- [Configuration](/getting-started/configuration) +- [Provider configuration](/providers/configuration) +- [Server settings](/operations/server-settings) +- [Troubleshooting](/guides/troubleshooting) diff --git a/docs/operations/monitoring.mdx b/docs/operations/monitoring.mdx index 932b16b..c71fb9c 100644 --- a/docs/operations/monitoring.mdx +++ b/docs/operations/monitoring.mdx @@ -1,35 +1,54 @@ --- title: Monitoring and operations -description: Monitor the Ferro Labs AI Gateway in production with Prometheus metrics, health checks, alerting rules, and operational runbooks to keep your LLM proxy healthy and performant. +description: Monitor Ferro Labs AI Gateway with authenticated Prometheus scraping, split /livez /readyz /health probes, and alert rules validated against the v1.4 metric surface. keywords: [AI gateway monitoring, LLM metrics, Prometheus AI gateway, production AI gateway, LLM proxy monitoring, gateway health checks] --- ## Metrics and health at a glance -| Signal | Endpoint | Format | -|---|---|---| -| Prometheus metrics | `GET /metrics` | Prometheus text | -| Deep health check | `GET /health` | JSON | -| Provider list | `GET /admin/providers` | JSON | +| Signal | Endpoint | Auth | Format | +|---|---|---|---| +| Prometheus metrics | `GET /metrics` | Bearer token, `read_only` or `admin` scope | Prometheus text | +| Liveness probe | `GET /livez` | None | JSON — process alive, no dependency checks | +| Readiness probe | `GET /readyz` | None | JSON — 200 when routable, 503 with a reason otherwise | +| Deep health check | `GET /health` | None | JSON — per-provider circuit state and model counts | +| Provider list | `GET /admin/providers` | Bearer token, `read_only` or `admin` scope | JSON | -See [Observability](/guides/observability) for the full metrics reference and log field docs. +`/metrics` sits behind the same scope check as every other read-only admin route — an unauthenticated scrape gets `401`. See [Observability](/guides/observability) for the full metrics reference and [Authentication](/guides/auth) for issuing keys. ## Prometheus scrape setup +`/metrics` requires a bearer token, so the scrape config needs credentials. Mint a dedicated `read_only` key rather than reusing `MASTER_KEY` — a scraping token that leaks in a Prometheus config file should never carry write access: + +```bash +curl -X POST http://gateway-host:8080/admin/keys \ + -H "Authorization: Bearer $MASTER_KEY" \ + -H "Content-Type: application/json" \ + -d '{"name": "prometheus-scraper", "scopes": ["read_only"]}' +# -> {"id": "...", "key": "fgw_...", "scopes": ["read_only"], ...} +``` + ```yaml # prometheus.yml scrape_configs: - - job_name: ferrogw + - job_name: ferro-ai-gateway + metrics_path: /metrics + scheme: http + authorization: + type: Bearer + credentials: fgw_the_key_returned_above static_configs: - targets: ["gateway-host:8080"] scrape_interval: 15s ``` +Without the `authorization` block every scrape 401s and the target shows `up == 0` in Prometheus with no other symptom — the most common first-deploy break. + ## Recommended alert rules ```yaml groups: - - name: ferrogw + - name: ferro-ai-gateway rules: # High error rate - alert: GatewayHighErrorRate @@ -54,7 +73,14 @@ groups: annotations: summary: "P99 request latency > 10s" - # Circuit breaker open + # Circuit breaker open. gateway_circuit_breaker_state is resolved from + # the live breakers at every scrape (0=closed 1=open 2=half_open) — it + # is never pushed, so this alert clears itself the moment the breaker + # leaves Open. A series exists only for a provider with circuit_breaker + # configured on at least one of its targets; a provider with no breaker + # emits no series at all, so this alert cannot fire for it (silent, not + # a false negative — check gateway_provider_errors_total instead for + # providers running without a breaker). - alert: GatewayCircuitBreakerOpen expr: gateway_circuit_breaker_state == 1 for: 1m @@ -63,14 +89,44 @@ groups: annotations: summary: "Circuit breaker open on {{ $labels.provider }}" - # All providers unhealthy - - alert: GatewayNoHealthyProviders - expr: up{job="ferrogw"} == 0 + # Provider errors feeding a breaker toward open. Only provider_error and + # timeout indicate an unhealthy upstream — circuit_open, client_canceled, + # backpressure, and plugin_error are the gateway declining or shedding, + # or the caller leaving, and alerting on those pages someone for traffic + # shape, not an outage. + - alert: GatewayProviderErrors + expr: | + sum by (provider) ( + rate(gateway_provider_errors_total{error_type=~"provider_error|timeout"}[5m]) + ) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "{{ $labels.provider }} returning errors or timing out" + + # No routable targets. Prometheus scraping /metrics can't reach a + # gateway with zero routable targets any differently than a healthy one + # (both answer up == 1), so this pairs with an external readyz probe — + # see "Load balancer and orchestrator health checks" below — rather + # than being derivable from metrics alone. + - alert: GatewayScrapeDown + expr: up{job="ferro-ai-gateway"} == 0 for: 1m labels: severity: critical annotations: - summary: "Gateway has no healthy providers" + summary: "Gateway not scrapeable" + + # MCP server down. 1->0 with no config change means the transport died + # (e.g. a stdio subprocess exited); alert on required servers first. + - alert: GatewayMCPServerDown + expr: gateway_mcp_server_up == 0 + for: 2m + labels: + severity: warning + annotations: + summary: "MCP server {{ $labels.server_name }} not ready" # Model catalog falling back to the embedded backup - alert: GatewayCatalogFallback @@ -82,9 +138,13 @@ groups: summary: "Model catalog using embedded fallback (remote source unreachable)" ``` +:::note +`gateway_circuit_breaker_state` and `gateway_mcp_server_up` are both gauges resolved fresh on every scrape — there's nothing to reset after an incident, and an absent series is informative (no breaker configured / no MCP servers registered), not a gap in data. +::: + ## Grafana dashboard -A community Grafana dashboard JSON is available in the repository at [`docs/grafana-dashboard.json`](https://github.com/ferro-labs/ai-gateway/blob/main/docs/). Import it into your Grafana instance and point the data source to your Prometheus server. +A ready-made Grafana dashboard ships in the repository at [`deploy/fullstack/grafana/dashboards/ferro-ai-gateway.json`](https://github.com/ferro-labs/ai-gateway/blob/main/deploy/fullstack/grafana/dashboards/ferro-ai-gateway.json). The fastest way to see it wired up end to end — gateway, Prometheus, Grafana, and Jaeger together — is `make up-fullstack` from the repo root ([`deploy/README.md`](https://github.com/ferro-labs/ai-gateway/blob/main/deploy/README.md)); import the JSON into an existing Grafana instance and point its data source at your Prometheus server otherwise. Key panels to build manually if you prefer: @@ -96,6 +156,8 @@ Key panels to build manually if you prefer: | Token usage / min | `(rate(gateway_tokens_input_total[1m]) + rate(gateway_tokens_output_total[1m])) * 60` | | Estimated spend / hour | `sum by (model) (rate(gateway_request_cost_usd_total[5m])) * 3600` | | Provider breakdown | `sum by (provider) (rate(gateway_requests_total[5m]))` | +| Provider error mix | `sum by (provider, error_type) (rate(gateway_provider_errors_total[5m]))` | +| MCP server availability | `gateway_mcp_server_up` | | Catalog loads by source | `sum by (source, result) (rate(gateway_catalog_loads_total[15m]))` | ## Logging pipeline @@ -104,17 +166,38 @@ Ship stdout JSON logs to your log aggregator: ```bash # Pipe to a log collector -./ferrogw 2>&1 | your-log-shipper --format=json +./ferrogw serve 2>&1 | your-log-shipper --format=json # Or use Docker logging drivers docker run ... --log-driver=awslogs ghcr.io/ferro-labs/ai-gateway:latest ``` -Filter gateway logs by `trace_id` in your aggregator to correlate all events for a single request across plugins and provider calls. +Filter gateway logs by `trace_id` in your aggregator to correlate all events for a single request across plugins and provider calls. The same value is echoed on the `X-Request-ID` response header (32 lowercase hex characters, no dashes) and, when tracing is on, equals the OpenTelemetry trace ID — one identifier across logs, the response header, and spans. See [Request logging](/operations/request-logging) for the persisted-log query API and column reference. ## Resiliency controls -- **Circuit breakers** — automatically exclude failing providers; see the `gateway_circuit_breaker_state` metric -- **Retries** — configurable per target with status-code filtering (`on_status_codes`) -- **Fallback strategy** — automatically promotes to the next target when primary fails -- **Health endpoint** — integrate into your load balancer's health check for automatic traffic shifting +- **Circuit breakers** — configured per target (`targets[].circuit_breaker`), one breaker per target shared across chat/streaming/embeddings/images; every routing strategy skips an open circuit. State is exposed on `gateway_circuit_breaker_state` and, with request context, on `GET /health`. +- **Retries** — configurable per target with status-code filtering (`retry.on_status_codes`), honored under every strategy including `single` (set `attempts: 1` to keep single-attempt behavior). +- **Fallback and pool strategies** — `fallback`, `loadbalance`, `least-latency`, `cost-optimized`, and `ab-test` all advance past a failed or open-circuit target automatically; `single`, `conditional`, and `content-based` commit to one target and report its outcome. See [Routing](/routing). + +## Load balancer and orchestrator health checks + +Point your load balancer or orchestrator readiness probe at **`/readyz`**, not `/health`. `/readyz` is the cheap, unauthenticated signal designed for this: it answers `200` when at least one configured target is routable, or `503` with a fixed reason (`no routable targets`, `store unreachable`, or `required mcp server unavailable`) otherwise, and it's cached for one second so a burst of probes never fans out to backing-store pings. `/health` is a deep diagnostic — per-provider circuit state and catalog model counts — meant for humans and dashboards, not for gating traffic. + +Use `/livez` for the orchestrator's *restart* decision (process alive, no dependency checks, always `200`) and keep it separate from `/readyz`'s *remove from rotation* decision — collapsing the two means a struggling-but-alive instance either never gets pulled from the load balancer or gets killed for a condition a restart can't fix (e.g. every provider's circuit open). + +```yaml +# Kubernetes example +readinessProbe: + httpGet: + path: /readyz + port: 8080 + periodSeconds: 10 +livenessProbe: + httpGet: + path: /livez + port: 8080 + periodSeconds: 10 +``` + +Alert on `gateway_mcp_server_up == 0` for any MCP server marked `required: true` in `mcp_servers[]` — that's the same signal gating `/readyz`, surfaced as a metric so it can page before an orchestrator starts cycling the instance. diff --git a/docs/operations/request-logging.mdx b/docs/operations/request-logging.mdx index b22f4bc..05d98e7 100644 --- a/docs/operations/request-logging.mdx +++ b/docs/operations/request-logging.mdx @@ -1,55 +1,125 @@ --- title: Request logging -description: Persist LLM request and response logs from the Ferro Labs AI Gateway to SQLite or PostgreSQL using the request-logger plugin for auditability and debugging. -keywords: [LLM request logging, AI gateway audit log, SQLite AI logs, Postgres LLM logs, AI observability logging, request logger plugin] +description: Configure the multi-stage request-logger plugin, persist rows to SQLite or Postgres, and query duration, TTFT, cost, and API key via the admin API. +keywords: [LLM request logging, AI gateway audit log, SQLite AI logs, Postgres LLM logs, AI observability logging, request logger plugin, admin logs api] --- -## Plugin-based logging +The `request-logger` plugin records every request the gateway handles — as structured stdout log lines always, and as rows in a shared request-log store when persistence is configured. The [admin API](#query-the-request-log) reads that same store, powering `GET /admin/logs`, `GET /admin/logs/stats`, and the dashboard's [Request Logs page](/guides/dashboard). -Use the `request-logger` plugin to record request lifecycle events. +## Enable the request-logger plugin + +`request-logger` is a **multi-stage** plugin: it needs one `plugins[]` entry per lifecycle stage it participates in, each carrying byte-identical config. Give it all three — `before_request`, `after_request`, and `on_error` — or logging is incomplete: + +- **Missing `after_request`**: the row written at `before_request` never gets its completion data. No `duration_ms`, `ttft_ms`, `cost_usd`, token counts, or provider — half the job. +- **Missing `on_error`**: a failed request never reaches `after_request`, so it produces no terminal row at all and vanishes from the default `/admin/logs` listing. + +Listing the same multi-stage plugin at only one stage isn't a smaller version of logging — it silently drops the rows the other stages were responsible for. ```yaml -- name: request-logger - type: logging - stage: before_request - enabled: true - config: - level: info - persist: true - backend: sqlite - dsn: ferrogw-requests.db +plugins: + - name: request-logger + type: logging + stage: before_request + enabled: true + config: + level: info + persist: true + - name: request-logger + type: logging + stage: after_request + enabled: true + config: + level: info + persist: true + - name: request-logger + type: logging + stage: on_error + enabled: true + config: + level: info + persist: true ``` -## Log store backends +The gateway compares each stage's `config` (name + JSON encoding) and refuses to start if any of the three disagree — that's what keeps the instances in sync rather than silently splitting state. + +:::note +`request-logger` is a `logging`-type plugin, so it **fails open**: if the log store is down, full, or unreachable, the request still completes. The write failure is logged as a warning rather than silently dropped, so an operator can tell the persisted trail went incomplete instead of trusting a log that quietly lost rows. +::: -You can also enable the request log store for the admin API. +| Key | Type | Default | Description | +|---|---|---|---| +| `level` | string | `info` | stdout log level (`debug`\|`info`\|`warn`\|`error`) for the request/response lines. `on_error` always logs at `error` regardless of this setting. | +| `persist` | bool | `false` | Write rows to the shared request-log store. Requires `REQUEST_LOG_STORE_BACKEND` / `REQUEST_LOG_STORE_DSN` to be set at the process level — without them, `persist: true` logs a startup warning and the plugin stays stdout-only. | + +`backend` and `dsn` keys inside the plugin's own `config` block are **obsolete** — they're ignored with a warning. Persistence targets and credentials are process-level configuration, set once via the environment variables below, not per plugin instance. + +A cache-served response is logged like any other request: it carries the real token usage but a real, measured `$0` cost (not `null` — see [nullable columns](#what-each-row-contains) below). + +## Configure the request-log store ```bash export REQUEST_LOG_STORE_BACKEND=sqlite export REQUEST_LOG_STORE_DSN=ferrogw-requests.db ``` -Supported backends are `sqlite` and `postgres`. +Supported backends are `sqlite` and `postgres`. This is the same store the plugin writes to and the admin API reads from — one shared store, not a per-plugin database. -When request log storage is not enabled, admin log endpoints return `501 not implemented`: +When the store isn't configured, admin log endpoints return `501 not implemented`: - `GET /admin/logs` - `GET /admin/logs/stats` - `DELETE /admin/logs` -## Admin log filters +## What each row contains + +| Column | Type | Notes | +|---|---|---| +| `duration_ms` | float, nullable | End-to-end time the gateway spent on the request, excluding the `after_request` plugin stage. | +| `ttft_ms` | float, nullable | Time to first token. Only measured for streaming requests — non-streaming rows carry `null`. | +| `cost_usd` | float, nullable | Estimated cost from the model catalog. **`null` means "unpriced"** — the catalog has no rate for that model — which is different from a real `$0.0` (an unlisted/free model, or a cache hit). Treating `null` as zero silently understates spend. | +| `api_key_id` | string | Opaque credential ID the request was served under (never the secret). Empty when the request carried no credential. | + +All timestamps are stored in UTC. + +## Query the request log `GET /admin/logs` supports: -- `limit` (default `50`, max `200`) -- `offset` (default `0`) -- `stage`, `model`, `provider` -- `since` (RFC3339) +| Filter | Description | +|---|---| +| `limit` | Default `50`, max `200`. | +| `offset` | Default `0`. | +| `stage` | `before_request`, `after_request`, `on_error`, or `all` for the raw per-stage stream. See below. | +| `model`, `provider` | Exact match. | +| `api_key_id` | Exact match against the credential's opaque ID. `api_key_id=none` selects rows with no credential — both unauthenticated requests and rows written before the column existed. | +| `since` | RFC3339 timestamp. | + +By default (no `stage` param), the endpoint returns **one row per request** — only terminal-stage rows (`after_request` for a completion, `on_error` for a failure). The plugin writes one row per stage it ran at, so without this default every request would appear listed twice: once from `before_request` with no completion data, once from its terminal stage. Pass `stage=all` to get the raw per-stage event stream instead, or `stage=` to see one specific stage. + +`DELETE /admin/logs` requires `before` (RFC3339), with optional `stage`, `model`, and `provider` filters (no `api_key_id` — a purge is scoped by time and dimension, not credential). + +## Request log statistics `GET /admin/logs/stats` supports: -- `limit` (top-N buckets, max `100`) -- `stage`, `model`, `provider` -- `since` (RFC3339) +| Filter | Description | +|---|---| +| `limit` | Top-N entries per dimension breakdown, max `100`. | +| `buckets` | Number of points in the time series, max `120`. Omit or `0` for no series. | +| `stage`, `model`, `provider` | Exact match. | +| `since` | RFC3339 timestamp. | + +The response includes: + +- **Summary**: total/error entry counts, `prompt_tokens` and `completion_tokens` split out separately, `cost_usd` total, and `unpriced_requests` — the count of requests the catalog couldn't price, so a `cost_usd` total isn't misread as the whole bill. +- **Percentiles**: `latency_ms` and `ttft_ms` distributions (p50, p95, p99, max, mean, count), computed from the stored `duration_ms`/`ttft_ms` values. `null` when nothing was measured in the window, rather than a misleading zero. +- **Dimension breakdowns**: `by_stage`, `by_provider`, `by_model` — each with count, errors, tokens, `cost_usd`, and `unpriced` per group, so you can see which model consumed the tokens, not just which was called most. +- **`top_errors`**: the most frequent distinct failure messages (fixed at 8). +- **`series`**: a time series with `buckets` points when requested, each carrying `requests`, `errors`, `prompt_tokens`, and `completion_tokens`; the response marks `truncated: true` if the series stopped short of the requested window. + +## Related -`DELETE /admin/logs` requires `before` (RFC3339), with optional `stage`, `model`, and `provider` filters. +- [request-logger plugin reference](/plugins/request-logger) +- [Dashboard](/guides/dashboard) — the Request Logs page built on this API +- [Server settings](/operations/server-settings) — `REQUEST_LOG_STORE_BACKEND` / `REQUEST_LOG_STORE_DSN` and other process configuration +- [Monitoring](/operations/monitoring) diff --git a/docs/operations/server-settings.mdx b/docs/operations/server-settings.mdx index 2463c2d..51d1911 100644 --- a/docs/operations/server-settings.mdx +++ b/docs/operations/server-settings.mdx @@ -1,13 +1,13 @@ --- title: Server settings -description: Complete runtime environment-variable reference for the Ferro Labs AI Gateway — config path, listen port, the MASTER_KEY credential, rate limiting, model-catalog override, pprof, and the security-sensitive dev toggles. -keywords: [AI gateway server settings, environment variables LLM proxy, gateway runtime config, gateway port settings, CORS AI gateway, MASTER_KEY, ALLOW_UNAUTHENTICATED_PROXY, FERRO_MODEL_CATALOG_URL] +description: Environment-variable reference for the AI Gateway — GATEWAY_ENV production mode, MASTER_KEY, rate limiting, CORS, catalog timeouts, pprof, and OTEL tracing. +keywords: [AI gateway server settings, environment variables LLM proxy, gateway runtime config, GATEWAY_ENV production mode, MASTER_KEY, RATE_LIMIT_RPS, CORS_ORIGINS, FERRO_MODEL_CATALOG_TIMEOUT] --- The gateway is configured entirely through environment variables (operational settings) and the [config file](/getting-started/configuration) (routing, plugins, tracing). This page is the reference for the environment variables read at startup. :::warning Security-sensitive settings -A few variables relax authentication or expose internals and must **never** be enabled in production. They are marked **(security-sensitive)** below: `MASTER_KEY`, `ALLOW_UNAUTHENTICATED_PROXY`, and `ENABLE_PPROF`. +A few variables relax authentication or expose internals and must **never** be enabled on a network-reachable deployment. They are marked **(security-sensitive)** below: `MASTER_KEY`, `ALLOW_UNAUTHENTICATED_PROXY`, and `ENABLE_PPROF`. Set `GATEWAY_ENV=production` (see [Production mode](#production-mode)) so the two most dangerous misconfigurations — an unauthenticated proxy and a `*` CORS origin — refuse to start instead of shipping silently. ::: ## Quick reference @@ -16,13 +16,18 @@ A few variables relax authentication or expose internals and must **never** be e |---|---|---| | `GATEWAY_CONFIG` | _(none)_ | Path to the JSON or YAML config file (format auto-detected) | | `PORT` | `8080` | HTTP listen port | -| `MASTER_KEY` | _(none)_ | **(security-sensitive)** Single admin + proxy credential; generate with `ferrogw init` | +| `GATEWAY_ENV` | _(none)_ | Set to `production` to turn on startup safety checks — see [Production mode](#production-mode) | +| `MASTER_KEY` | _(none)_ | **(security-sensitive)** Bootstrap/break-glass admin + proxy credential; generate with `ferrogw init` | | `ALLOW_UNAUTHENTICATED_PROXY` | `false` | **(security-sensitive)** Disables `/v1` auth — local dev only | -| `ENABLE_PPROF` | `false` | **(security-sensitive)** Exposes `/debug/pprof` profiling endpoints | -| `RATE_LIMIT_RPS` | _(off)_ | Per-IP requests/second; enables the rate-limit middleware when set | -| `RATE_LIMIT_BURST` | `0` | Per-IP burst capacity for the rate-limit middleware | +| `CORS_ORIGINS` | _(none)_ | Comma-separated allowed origins, matched **literally** (no wildcard) | +| `TRUSTED_PROXIES` | loopback | Comma-separated CIDRs whose `X-Forwarded-For`/`X-Real-IP` are honored for client-IP resolution | +| `RATE_LIMIT_RPS` | `20` | Per-IP requests/second; the middleware is **on by default** — `0` disables it | +| `RATE_LIMIT_BURST` | `40` | Per-IP burst capacity | +| `ENABLE_PPROF` | `false` | **(security-sensitive)** Exposes `/debug/pprof` profiling endpoints (admin scope required) | | `FERRO_MODEL_CATALOG_URL` | GitHub releases | Override the model-catalog source (air-gapped / custom pricing) | -| `CORS_ORIGINS` | _(none)_ | Comma-separated list of allowed origins | +| `FERRO_MODEL_CATALOG_TIMEOUT` | `10s` | Bounds the startup catalog fetch; `0` skips the remote fetch entirely | +| `FERRO_MODEL_DISCOVERY_INTERVAL` | _(off)_ | Opt-in live `/models` refresh interval (Go duration, e.g. `6h`) | +| `FERRO_OLLAMA_MODELS` | _(none)_ | Comma-separated Ollama model list narrowing `/v1/models` | | `LOG_LEVEL` | `info` | `debug`, `info`, `warn`, `error` | | `LOG_FORMAT` | `json` | `json` or `text` (human-readable for dev) | @@ -30,58 +35,94 @@ A few variables relax authentication or expose internals and must **never** be e - `GATEWAY_CONFIG` - path to the JSON or YAML config file (format auto-detected). When unset the gateway boots from environment-derived defaults (a fallback strategy over every provider whose key is present). - `PORT` - HTTP listen port (default `8080`). -- `CORS_ORIGINS` - comma-separated list of allowed origins. - `LOG_LEVEL` - logging level (`debug`, `info`, `warn`, `error`; default `info`). - `LOG_FORMAT` - log output format; set `LOG_FORMAT=text` for human-readable development logs. ## Credentials and auth -- `MASTER_KEY` - **(security-sensitive)** the single bearer credential the gateway accepts for both admin (`/admin/*`) and proxy (`/v1/*`) requests. Generate one with `ferrogw init`. If unset, the gateway starts but logs a warning and admin routes have no valid credential. Treat it like a root password — never commit it, and rotate it if exposed. -- `ALLOW_UNAUTHENTICATED_PROXY` - **(security-sensitive)** when set to `true`, the `/v1/*` proxy and inference routes skip authentication entirely. This is a **development-only** convenience; the gateway logs a warning at startup when it is set. Never enable it on a network-reachable deployment. +- `MASTER_KEY` - **(security-sensitive)** the bootstrap and break-glass admin credential the gateway accepts for both admin (`/admin/*`) and proxy (`/v1/*`) requests. Generate one with `ferrogw init`. It has no key-store row, so unlike a stored key it cannot be revoked or expired without restarting the process — treat it like a root password. If unset, the gateway starts but logs a warning and admin routes have no valid credential. +- `ALLOW_UNAUTHENTICATED_PROXY` - **(security-sensitive)** when set to `true`, the `/v1/*` proxy and inference routes skip authentication entirely. This is a **development-only** convenience; the gateway logs a warning at startup when it is set, and `GATEWAY_ENV=production` refuses to start with it enabled. -### Admin bootstrap (deprecated) +`ADMIN_BOOTSTRAP_KEY`, `ADMIN_BOOTSTRAP_READ_ONLY_KEY`, and `ADMIN_BOOTSTRAP_ENABLED` were removed in v1.4.0 ; `MASTER_KEY` is the only bootstrap credential now. -- `ADMIN_BOOTSTRAP_KEY` - first-run admin bearer key. **Deprecated** — use `MASTER_KEY`. -- `ADMIN_BOOTSTRAP_READ_ONLY_KEY` - first-run read-only bearer key. **Deprecated** — use `MASTER_KEY`. -- `ADMIN_BOOTSTRAP_ENABLED` - enable or disable bootstrap keys (default enabled). +:::tip Per-operator keys +Give each operator their own admin-scoped key via `POST /admin/keys` (or the dashboard) instead of sharing `MASTER_KEY` day to day — a shared key can't be revoked for one person without rotating it for everyone, and credential-change records all name the same key. Reserve `MASTER_KEY` for bootstrap and break-glass recovery. See [Auth](/guides/auth). +::: + +## Production mode + +`GATEWAY_ENV=production` (case-insensitive) turns on a fixed set of startup checks, reported together so fixing a deployment is one edit rather than one restart per problem: + +| Setting | Outcome under `GATEWAY_ENV=production` | +|---|---| +| `ALLOW_UNAUTHENTICATED_PROXY=true` | **Refused** — startup exits; every `/v1/*` data-plane endpoint would be unauthenticated | +| `CORS_ORIGINS` containing `*` | **Refused** — matched literally, so it denies every cross-origin request while reading as an allow-all | +| `RATE_LIMIT_RPS=0` | **Warned** — startup continues; a normal choice when limits are enforced by an ingress or upstream API gateway | +| `ENABLE_PPROF=true` | **Warned** — profiling routes stay behind the admin scope; the risk is leaving them mounted past an incident | +| In-memory `API_KEY_STORE_BACKEND` | **Warned** — operator keys, dashboard sessions, and the audit trail are lost on restart, leaving `MASTER_KEY` as the only way back in | -Bootstrap keys are only honored while the API key store is empty. +Outside production (`GATEWAY_ENV` unset or any other value), all five settings are honored with only a startup warning — none of them is refused. -## Stores +## CORS and trusted proxies -- `CONFIG_STORE_BACKEND` - `memory`, `sqlite`, `postgres` -- `CONFIG_STORE_DSN` - SQLite file path or Postgres DSN -- `API_KEY_STORE_BACKEND` - `memory`, `sqlite`, `postgres` -- `API_KEY_STORE_DSN` - SQLite file path or Postgres DSN -- `REQUEST_LOG_STORE_BACKEND` - `sqlite`, `postgres` -- `REQUEST_LOG_STORE_DSN` - SQLite file path or Postgres DSN +- `CORS_ORIGINS` - comma-separated list of allowed origins, matched **literally** against the request's `Origin` header — there is no wildcard, so a `CORS_ORIGINS=*` entry allows nothing a browser would ever send (quotes are stripped before the check, so `CORS_ORIGINS="*"` is caught too). List each origin explicitly. Unset serves no CORS headers. An unlisted origin is denied by the **absence** of `Access-Control-Allow-Origin`, not by a refused preflight. +- `TRUSTED_PROXIES` - comma-separated CIDRs of trusted reverse proxies; `X-Forwarded-For`/`X-Real-IP` is honored only from these (default: loopback, `127.0.0.0/8` and `::1/128`). An invalid value exits at startup. This matters for the per-IP rate limiter below — without the real proxy's CIDR listed, every request behind an untrusted proxy resolves to the same client IP and shares one bucket. ## Rate limiting -- `RATE_LIMIT_RPS` - per-IP requests per second. Setting a positive value enables the per-IP token-bucket rate-limit middleware; leaving it unset (or non-positive) disables it. -- `RATE_LIMIT_BURST` - per-IP burst capacity (default `0`). +Per-IP rate limiting is **enabled by default** — it is not opt-in. + +- `RATE_LIMIT_RPS` - per-IP requests per second (default `20`). Set to `0` to disable the middleware entirely; an invalid or negative value is ignored with a startup warning and the default is used instead. Setting `RATE_LIMIT_RPS` alone also resets the burst back to the default `40` — pair it with `RATE_LIMIT_BURST` for a custom rate/burst combination. +- `RATE_LIMIT_BURST` - per-IP burst capacity (default `40`). The limiter tracks at most 100,000 IPs. Rejected requests are surfaced on the `gateway_rate_limit_rejections_total` metric — see [Monitoring](/operations/monitoring). +:::tip Not the same knob as the rate-limit plugin +`RATE_LIMIT_RPS=0` disables this per-IP middleware. The **`rate-limit` plugin**'s `requests_per_second: 0` means the opposite — a rate the gateway cannot serve — and is rejected at config load; disable that plugin with `enabled: false` instead. See [Rate limiting](/guides/rate-limiting). +::: + ## Model catalog -- `FERRO_MODEL_CATALOG_URL` - override the model-catalog source URL. The catalog supplies pricing, capabilities, and lifecycle metadata used for cost estimation and `/v1/models` enrichment. By default the gateway fetches the catalog from the public GitHub releases of `ferro-labs/model-catalog` (with a 1-second timeout and an embedded backup as fallback), so the gateway **never fails to start** if the source is unreachable. Point this at an internal mirror for **air-gapped** deployments, or at a custom catalog for **enterprise/custom pricing**. Each load attempt is recorded on the `gateway_catalog_loads_total` metric (`source`, `result` labels). Any userinfo and query parameters in the URL are stripped from logs, so tokens embedded in the URL are not leaked. +- `FERRO_MODEL_CATALOG_URL` - override the model-catalog source URL. The catalog supplies pricing, capabilities, and lifecycle metadata used for cost estimation and `/v1/models` enrichment. By default the gateway fetches the catalog from the public GitHub releases of `ferro-labs/model-catalog`, with an embedded catalog as fallback, so the gateway **never fails to start** if the source is unreachable. Point this at an internal mirror for **air-gapped** deployments, or at a custom catalog for **enterprise/custom pricing**. Any userinfo and query parameters in the URL are stripped from logs, so tokens embedded in the URL are not leaked. +- `FERRO_MODEL_CATALOG_TIMEOUT` - Go duration bounding the startup catalog fetch (default `10s`). The fetch runs before the listener binds, so a blocked-egress deployment waits this long before falling back to the embedded catalog. Set `0` to skip the remote fetch entirely — the way to keep a fully air-gapped gateway from spending any startup time on it. +- `FERRO_MODEL_DISCOVERY_INTERVAL` - opt-in Go duration (e.g. `6h`) to live-refresh model lists from provider `/models` endpoints after startup. Unset, unparseable, or under `1m` leaves discovery disabled. + +Each load attempt is recorded on the `gateway_catalog_loads_total` metric (`source`, `result` labels). + +## Ollama model list + +- `FERRO_OLLAMA_MODELS` - comma-separated Ollama model list narrowing what `/v1/models` advertises. Neither variable is required — Ollama serves whatever the operator pulled onto it, so an unset list lets the provider serve any model. +- `OLLAMA_MODELS` - **deprecated**, read for one more release with a startup warning. This is Ollama's own variable for the models *directory*, not a model list; a path-shaped value (`/…`, `~…`, `./…`) with no comma is dropped with a warning rather than registered as a bogus model id. `FERRO_OLLAMA_MODELS` wins when both are set. ## Profiling -- `ENABLE_PPROF` - **(security-sensitive)** when set to `1`, `true`, or `yes`, mounts the Go `net/http/pprof` handlers under `/debug/pprof` (heap, goroutine, profile, trace, etc.). These endpoints expose runtime internals and can be abused for denial of service; keep them off in production or restrict them to an internal network. +- `ENABLE_PPROF` - **(security-sensitive)** when set to `true`, mounts the Go `net/http/pprof` handlers under `/debug/pprof` (heap, goroutine, profile, trace, etc.), alongside `/debug/vars` (expvar). Everything under `/debug` requires a bearer token with the **admin** scope — a heap or trace profile is a memory image that can contain request bodies and credentials, and expvar publishes the process command line. `GATEWAY_ENV=production` allows `ENABLE_PPROF` but warns at startup; unmount it once an investigation ends. ## Bedrock / AWS credentials -AWS Bedrock is registered automatically when `AWS_REGION` or `AWS_ACCESS_KEY_ID` is present. +AWS Bedrock is registered automatically when `AWS_REGION`, `AWS_ACCESS_KEY_ID`, or `AWS_BEARER_TOKEN_BEDROCK` is present. - `AWS_REGION` - AWS region for Bedrock. - `AWS_ACCESS_KEY_ID` - AWS access key (optional — falls back to the instance role / default credential chain). - `AWS_SECRET_ACCESS_KEY` - AWS secret key. - `AWS_SESSION_TOKEN` - session token for **temporary credentials** (e.g. STS / assumed-role / SSO sessions). Required alongside the access key and secret when using temporary credentials. +- `AWS_BEARER_TOKEN_BEDROCK` - a Bedrock bearer token, used instead of SigV4 keys. -For the full list of provider API-key variables (OpenAI, Anthropic, Gemini, and the rest of the 30 supported providers), see the [Providers guide](/guides/providers). +For the full list of provider API-key variables (OpenAI, Anthropic, Gemini, and the rest of the 30 supported providers), see [Provider configuration](/providers/configuration). ## Tracing (OTEL_*) -Standard OpenTelemetry environment variables — `OTEL_EXPORTER_OTLP_ENDPOINT`, `OTEL_EXPORTER_OTLP_HEADERS`, `OTEL_TRACES_SAMPLER`, `OTEL_TRACES_SAMPLER_ARG` — configure trace export and **take precedence** over the `observability.tracing` config block. See [Observability](/guides/observability) for details. +The gateway reads exactly two `OTEL_*` variables — nothing else: + +- `OTEL_EXPORTER_OTLP_ENDPOINT` - OTLP collector **base** endpoint. Setting it alone turns tracing on and takes precedence over the config file's `observability.tracing.endpoint`. The base gets `v1/traces` appended under `http/protobuf`. +- `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` - signal-specific traces endpoint, used verbatim (no path appended). It outranks `OTEL_EXPORTER_OTLP_ENDPOINT` when both are set. + +Everything else about tracing — including the sampler — is **config-only**. `observability.tracing.sample_ratio` builds a `ParentBased` head sampler (default `1.0`, so an inbound sampled trace is always followed); `OTEL_TRACES_SAMPLER` and `OTEL_TRACES_SAMPLER_ARG` have **no effect** on this gateway. `OTEL_EXPORTER_OTLP_HEADERS` reaches the exporter through the OTel SDK directly, not through the gateway. See the [config reference](/getting-started/configuration) for the full `observability.tracing` block and [Monitoring](/operations/monitoring) for metrics and the OTLP exporter setup. + +## Related + +- [Configuration](/getting-started/configuration) — routing, plugins, and the `observability.tracing` config block +- [Monitoring](/operations/monitoring) — metrics, health checks, and OTLP tracing setup +- [Auth](/guides/auth) — MASTER_KEY, API keys, and dashboard sessions +- [Rate limiting](/guides/rate-limiting) — the per-IP middleware vs. the `rate-limit` plugin +- [Provider configuration](/providers/configuration) — per-provider API keys and `_BASE_URL` diff --git a/docs/plugins/budget.mdx b/docs/plugins/budget.mdx new file mode 100644 index 0000000..874cee9 --- /dev/null +++ b/docs/plugins/budget.mdx @@ -0,0 +1,82 @@ +--- +title: Budget Plugin +description: Reference for the budget built-in plugin — per-API-key USD spend caps, 402 insufficient_quota on exhaustion, operator-set token rates, and config keys. +keywords: [budget plugin, AI gateway spend limit, insufficient_quota, 402, per-key budget, cost cap, LLM spend limit, token pricing] +--- + +The `budget` plugin enforces a per-API-key USD spend cap using in-memory accumulation. It runs at both `before_request` (check accumulated spend against the limit) and `after_request` (price the completed request from token usage and add it to the store). + +## Facts + +| | | +|---|---| +| Stages | `before_request` **and** `after_request` — multi-stage, byte-identical config required at both | +| Reported type | `ratelimit` (`plugin.TypeRateLimit`) | +| Failure policy | Fails **closed** (a plugin error returns `500`) | +| Denial status | **402 `insufficient_quota`** — not 429. Decided by plugin name in `internal/apierror`, checked *before* the generic `ratelimit` → 429 branch, regardless of the reported type | +| Cap type | **Soft** cap, no reservation — concurrent in-flight requests for the same key can collectively overshoot | +| Agentic loops | Re-checked on every MCP tool-loop turn (`RunBeforeLoopTurn`) — can stop an overspending loop mid-request | +| Pricing source | The plugin's **own configured rates** (`input_per_m_tokens` / `output_per_m_tokens` / cache rates) — **not** the model catalog. See [Gotchas](#gotchas) | +| External dependencies | None — in-memory (`sync.Map` of per-store spend maps), per-process. Spend does not survive a restart | + +## Config keys + +| Key | Type | Default | Required | Description | +|---|---|---|---|---| +| `store_id` | string | `"default"` | No | Shared spend-store key. Instances with the same `store_id` share one set of counters — how the `before_request` and `after_request` entries stay in sync. | +| `spend_limit_usd` | float `>= 0` | `0` (unlimited) | No | Max cumulative USD per API key. `0` disables the check. Setting it `> 0` while every rate below is `0` is a **load error** — cost would always compute to `0` and the cap could never be enforced. | +| `input_per_m_tokens` | float | `0` | No | USD per 1M prompt tokens. | +| `output_per_m_tokens` | float | `0` | No | USD per 1M completion tokens. `ReasoningTokens` are a subset of `CompletionTokens` — already covered, never billed again. | +| `cache_read_per_m_tokens` | float `>= 0` (pointer — unset is distinct from `0`) | unset | No | USD per 1M cached prompt tokens. **Unset ≠ `0`**: unset bills the whole prompt (including the cached subset) at `input_per_m_tokens`; `0.0` makes cached tokens free. When set, the cached subset comes off the input-rate count so it is billed once, not twice. | +| `cache_write_per_m_tokens` | float `>= 0` | unset (free) | No | USD per 1M cache-write tokens. `CacheWriteTokens` sit outside `PromptTokens` and bill only when this is set. | +| `max_keys` | int `>= 0` | `10000` | No | Max API keys tracked in memory per store. `0` = unlimited. At the cap, the **lowest-spend** key is evicted to make room and restarts at `$0` — under churn above `max_keys` distinct keys, a key's cap can be silently reset. | + +## Minimal config + +Both stage entries must carry byte-identical `config` — the gateway resolves them to one shared spend store by `name` + JSON-encoded config, and refuses to start if the two disagree. + +```yaml +plugins: + - name: budget + type: guardrail + stage: before_request + enabled: true + config: + store_id: default + spend_limit_usd: 10.0 + input_per_m_tokens: 3.0 + output_per_m_tokens: 15.0 + max_keys: 10000 + + - name: budget + type: guardrail + stage: after_request + enabled: true + config: + store_id: default + spend_limit_usd: 10.0 + input_per_m_tokens: 3.0 + output_per_m_tokens: 15.0 + max_keys: 10000 +``` + +## Gotchas + +- **402, not 429, on exhaustion.** Waiting does not restore a spend cap — only cost roll-off (there is none, in-memory spend never decays) or an explicit reset clears it. A `429` is retryable in every OpenAI SDK's default policy and would send a client into a backoff schedule that re-asks the same unanswerable question, roughly once a second, forever. `402 insufficient_quota` carries no `Retry-After` and is outside every SDK's retry set, so the first refusal is the last request the SDK makes on its own. +- **Soft cap — no reservation.** The `before_request` check is read-only: it reads already-committed spend, places no hold, and lets the request proceed. A bounded number of concurrent requests for the same key can all pass the check simultaneously and collectively push spend past the limit once each completes. The overshoot is bounded by in-flight-request-count × per-request cost, not unbounded. A hard cap (pre-authorization/reservation) is deliberately out of scope: a reservation that leaks on every error, cancellation, circuit-trip, or rejection would permanently pin a key at its cap. +- **Prices from operator-set rates, not the model catalog.** Unlike the cost figures shown elsewhere in the gateway (the `ferro.cost.usd` trace attribute, catalog-based cost visibility — see [Cost tracking](/guides/cost-tracking)), the budget plugin has no catalog to read from: it multiplies token counts by whatever `input_per_m_tokens` / `output_per_m_tokens` / cache rates you configure. If those rates drift from your real per-model pricing, the spend this plugin tracks drifts from your real bill. +- **Cache tokens: unset ≠ zero.** Leaving `cache_read_per_m_tokens` unset bills the *entire* prompt — cached subset included — at the input rate (a visible over-report beats a silent under-report of an unpriced dimension). Set it to `0.0` explicitly to make cached tokens free. When set to any value, `PromptTokens` (which is inclusive of `CacheReadTokens`) has the cached subset subtracted before the input rate applies, so cached tokens are never billed twice. `CacheWriteTokens` are outside `PromptTokens` entirely and cost nothing unless `cache_write_per_m_tokens` is set. +- **Two rates only — other billing dimensions accrue $0.** Only prompt and completion (plus the two optional cache rates) are priced. Most image providers report no token usage at all, so image-only traffic never touches a budget (only token-billed models like the `gpt-image` family accrue anything). Audio input (per-minute) and audio output (per-character) are not represented either — the shortfall is always an under-report, never an over-report. +- **Per-turn enforcement on agentic loops.** Inside an MCP tool loop, the gateway re-runs the `before_request` check on every turn and adds the request's own running spend (`pctx.Measurements.CostUSD`, when `HasCost` is set) to what the store already has on record. Without that term, a key sitting at 99% of its cap on turn one would get an entire loop regardless of how many turns it ran or how large the context grew, because the store itself is only written once, after the whole request completes. This is the one place a single request can close its own cap mid-flight. +- **No API key, no tracking.** The key is read from `Metadata["api_key"]` (the opaque credential id). A request with no key in metadata — e.g. running with `ALLOW_UNAUTHENTICATED_PROXY=true` — skips budget tracking entirely: it is neither rejected nor recorded against any key. +- **A cache hit records nothing.** `after_request` is skipped when `SkipProvider` is set (the response came from `response-cache`, not a provider) — there is no cost to record for a call that never left the process. This is also why the same prompt served from cache a hundred times does not consume a hundred times its cost from the budget. +- **In-memory only.** Spend does not survive a process restart, and a store is per-process — a multi-replica deployment gets independent counters per instance, not one shared budget. `ResetStore` / `ResetStoreKey` (Go API, not an admin endpoint) exist for housekeeping, e.g. clearing spend on key rotation. +- **`ferrogw validate` catches the same errors `Init` would.** `spend_limit_usd` set with every rate at `0`, or a negative `spend_limit_usd`/cache rate/`max_keys`, fails validation and startup — it is not a silently-inert config. + +## Related + +- [Cost tracking](/guides/cost-tracking) +- [Plugins overview](/plugins) +- [Response cache plugin](/plugins/response-cache) +- [Rate limit plugin](/plugins/rate-limit) +- [MCP](/guides/mcp) diff --git a/docs/plugins/enterprise.mdx b/docs/plugins/enterprise.mdx new file mode 100644 index 0000000..1878179 --- /dev/null +++ b/docs/plugins/enterprise.mdx @@ -0,0 +1,25 @@ +--- +title: Ferro Labs Managed Plugins +description: "Overview of the five Ferro Labs Managed security plugins — PII redaction, secret scanning, prompt shielding, schema guard, regex guard — not in the OSS gateway." +keywords: [Ferro Labs Managed plugins, PII redaction, secret scanning, prompt injection shield, schema guard, regex guard, AI gateway security] +--- + +Ferro Labs Managed adds five security plugins on top of the [6 open-source plugins](/plugins) that ship with the AI Gateway. They slot into the same `before_request`/`after_request` plugin pipeline, but the detection logic (PII models, secret patterns, injection scoring, schema/regex engines) is closed-source and only runs in managed deployments. + +| Plugin | What it guards | Typical action | +|---|---|---| +| `pii-redact` | Personally identifiable information in request or response content | Redact in place, or block the request | +| `secret-scan` | Leaked API keys, credentials, and tokens in request content | Block, or warn and pass through | +| `prompt-shield` | Prompt injection and jailbreak attempts in user messages | Block above a confidence threshold | +| `schema-guard` | Model output conformance to a JSON Schema | Block non-conforming responses | +| `regex-guard` | Custom regex-defined patterns (SSNs, business rules) | Block, or warn and pass through | + +:::info Ferro Labs Managed +These five plugins are **not included in the open-source build** — they are only available on [Ferro Labs Managed](/ferrocloud/overview). Join the [waitlist](https://www.ferrolabs.ai/) for access. +::: + +## Related + +- [Plugins overview](/plugins) — the 6 OSS plugins and the shared pipeline stages +- [Ferro Labs Managed overview](/ferrocloud/overview) +- [OSS vs. Ferro Labs Managed](/guides/oss-vs-ferrocloud) diff --git a/docs/plugins/max-token.mdx b/docs/plugins/max-token.mdx new file mode 100644 index 0000000..4fa5aef --- /dev/null +++ b/docs/plugins/max-token.mdx @@ -0,0 +1,61 @@ +--- +title: Max Token Plugin +description: Reference for the max-token guardrail plugin — rejects requests over a declared completion ceiling, message count, or input length; config keys, YAML, gotchas. +keywords: [max-token plugin, AI gateway guardrail, max_tokens limit, max_completion_tokens, message count limit, input length limit, request rejection, before_request] +--- + +`max-token` is a guardrail plugin that **rejects** a request whose declared completion ceiling exceeds `max_tokens`, whose message count exceeds `max_messages`, or whose total input length exceeds `max_input_length`. It runs at `before_request`, so a rejected request never reaches a provider. + +## Facts + +| | | +|---|---| +| Stage(s) | `before_request` only | +| Reported type | `guardrail` ([`plugin.TypeGuardrail`](/plugins)) | +| Failure policy | Fails **closed** — a plugin error (not a rejection) aborts the request as `500` | +| Denial status | `400 invalid_request_error` (`request_rejected`) — not `429` and not `402` | +| Multi-stage | No — a single `before_request` entry is a complete, valid configuration | +| Agentic loops | Re-checked on every MCP tool-loop turn (`RunBeforeLoopTurn`), since guardrails are not exempted the way `transform`/`logging`/`metrics` plugins are — each turn's completion ceiling and message count are enforced again | +| Content-reading | [`ContentAgnostic`](#gotchas) — reads request content only when `max_input_length` is configured above `0`; the other two checks are counts, not content | +| External dependencies | None — pure in-process arithmetic over the already-decoded request, no state carried between requests | + +## Configuration + +| Key | Type | Default | Required | Description | +|---|---|---|---|---| +| `max_tokens` | int (or float64) | `4096` | No | Rejects when the request's **effective** completion ceiling — `EffectiveMaxTokens()`, where `max_completion_tokens` supersedes `max_tokens` when both are set — exceeds this. `0` disables the check. A request that sets neither field declares no ceiling and passes uncapped; see [Gotchas](#gotchas). | +| `max_messages` | int (or float64) | `100` | No | Rejects when `len(Request.Messages)` exceeds this. `0` disables the check. Skipped on projected surfaces (embeddings/images set `Metadata["surface"]`), so a 150-document embedding batch is not refused as 150 "messages". | +| `max_input_length` | int (or float64) | `0` (off) | No | Rejects when the total character length of every message — `Content`, or for a multipart message the sum of `ContentParts[].Text` and each `ImageURL.URL` — exceeds this. `0` means no limit. Setting this above `0` makes the plugin read request content; see [Gotchas](#gotchas). | + +Config values are accepted only as `int` or `float64` (YAML/JSON both decode plain numbers to one of these); a value of any other type — a quoted string, for instance — is silently ignored and the key keeps its default. There is no `ferrogw validate` check for this: `max-token` does not implement `ConfigValidator`. + +## Minimal config + +```yaml +plugins: + - name: max-token + type: guardrail + stage: before_request + enabled: true + config: + max_tokens: 4096 + max_messages: 100 + max_input_length: 0 +``` + +## Gotchas + +- **It's reject-only — it never imposes a ceiling.** A request that sets neither `max_tokens` nor `max_completion_tokens` declares no ceiling, is not rejected, and runs to the provider's own default uncapped. This is deliberate: writing a ceiling the caller never asked for is transform behavior in a guardrail — the injected value would change what is sent upstream, show up in the provider's bill, and truncate completions nobody configured. An operator who needs a hard bound sets it on the client or picks a model whose default is the bound they want. +- **`max_completion_tokens` supersedes `max_tokens`.** The plugin reads `EffectiveMaxTokens()`, never `Request.MaxTokens` directly, so a request cannot pair a small `max_tokens` with a huge `max_completion_tokens` and have the huge value smuggled past the cap — a request setting `max_tokens: 5` and `max_completion_tokens: 500000` is rejected against the 500000 figure, not the 5. +- **`0` disables each check — opposite polarity to `rate-limit`.** For `max_tokens`, `max_messages`, and `max_input_length`, `0` means "this check is off." That's the reverse of the [`rate-limit`](/plugins/rate-limit) plugin, where `requests_per_second: 0` is rejected as a load error rather than treated as "no limiting." Use `0` here freely; disable `max-token` entirely with `enabled: false` instead if you want no checks at all. +- **`max_messages` is skipped on projected surfaces.** Embeddings and image-generation requests reach the plugin through a projection that turns each input element into one user message, so a 150-document embedding batch arrives here as 150 "messages." The gateway sets `Metadata["surface"]` on those requests, and `max-token` skips the message-count check when that key is present — a conversation-turn ceiling isn't a statement about how many documents may be embedded at once. `max_input_length`, a size measure, still applies on projected surfaces. +- **Content-agnostic by default; `max_input_length > 0` flips that.** With `max_input_length` unset (or `0`), `max-token` never reads message content — `max_tokens` is the caller's own declared number and `max_messages` is a list length, neither of which changes when the text inside a message can't be read. That matters for the `/v1/*` pass-through proxy: a surface that hands a guardrail an unreadable body (a multipart upload, an audio payload, token-ID input) has to treat a guardrail's vacuous approval as consent unless the guardrail says otherwise, so it refuses uninspectable pass-through bodies by default — *except* when the only configured `before_request` guardrail declares it reads no content, which `max-token` does (its `IgnoresRequestContent()` returns `true`) as long as `max_input_length` is `0`. Set `max_input_length` above `0` and the plugin becomes content-reading: it now measures the projected text, so an unreadable pass-through body would satisfy the cap at length zero — a vacuous pass, not a real one — and the gateway refuses those bodies instead of serving them once this key is set. +- **Multipart messages are measured from `ContentParts` alone.** `Content` already holds the concatenated text parts for a decoded multipart message, so `max_input_length` sums `ContentParts[].Text` plus each part's `ImageURL.URL` length rather than adding `Content` on top — adding both would double-count the text while still ignoring the image payload, which for a base64 data URI is nearly the entire request. +- **`Execute` does not branch on stage.** Nothing stops a config from registering `max-token` at `after_request` or `on_error` too, and the manager will call it there — `Execute` just re-runs the same three checks against `pctx.Request`, which is still populated in every stage. On a request that already cleared `before_request`, that second run is redundant (it re-approves what it already approved) rather than harmful, but it is not a documented pairing the way `budget`'s two stages are — there's nothing for a later stage to add. + +## Related + +- [Plugins overview](/plugins) +- [Word filter guardrail](/plugins/word-filter) +- [Rate limit plugin](/plugins/rate-limit) +- [Getting started: configuration](/getting-started/configuration) diff --git a/docs/plugins/overview.mdx b/docs/plugins/overview.mdx new file mode 100644 index 0000000..4fd0147 --- /dev/null +++ b/docs/plugins/overview.mdx @@ -0,0 +1,154 @@ +--- +title: Plugin system +description: "How plugins hook the Ferro Labs AI Gateway pipeline at before_request, after_request and on_error stages, with global config, ordering and failure policy." +keywords: [AI gateway plugins, plugin pipeline, before_request, after_request, on_error, fail-open fail-closed, response cache, rate limit, budget, request logger] +slug: /plugins +--- + +Plugins are the Ferro Labs AI Gateway's middleware. Each one runs at fixed points in the request lifecycle and can inspect the request, mutate it, deny it, skip the provider call, or record what happened. The six built-in plugins — guardrails, a rate limiter, a spend budget, a response cache, and a request logger — are all built on the same interface an out-of-tree plugin uses, so nothing about the built-ins is privileged. + +## What a plugin is + +A plugin is a named component implementing `plugin.Plugin` (`Name`, `Type`, `Init`, `Execute`, `Close`). The gateway loads it by name at startup, calls `Init` once with its `config` map, and then calls `Execute` on every request at each stage the plugin is registered for. `Execute` receives a `plugin.Context` carrying the request, the response (once available), per-request metadata, and the current stage — and can set `Reject`, `SkipProvider`, or mutate the request to influence the pipeline. + +A plugin's `Type()` — `guardrail`, `ratelimit`, `transform`, `logging`, `metrics`, or `auth` — is what the gateway acts on. It decides the failure policy, whether the plugin re-runs inside an agentic tool loop, and whether an admission check stands down. The `type:` written in your config is documentation only. + +## Plugins are global + +There is one plugin list, at the top level of the config. There is **no** per-route, per-target, or per-model plugin field — a plugin that is enabled is enabled for every routed request. + +```yaml +plugins: + - name: word-filter + type: guardrail # informational; the gateway reads the plugin's own Type() + stage: before_request + enabled: true # enabled: false skips registration entirely + config: + blocked_words: ["password", "secret"] + case_sensitive: false +``` + +Each entry is `{ name, type, stage, enabled, config }`. `enabled: false` removes the entry from the pipeline — it is never constructed. Unknown keys inside a `config` block are silently ignored (each plugin reads its own map), while the top-level config decoder is strict and rejects an unknown key. Because a mistyped setting does nothing rather than erroring, check the exact keys a plugin reads against the catalog (below) — `requests_per_minute: 1` leaves `rate-limit`'s `100/s` default in place. + +## The three stages + +A plugin declares which stage it runs in with `stage:`. The gateway runs the plugins of each stage in the order they appear in the config file. + +| Stage | When it runs | What it can do | What it can't | +|---|---|---|---| +| `before_request` | After authentication, before the provider is called | Inspect and mutate the request; `Reject` it; set `SkipProvider` to serve a cached/synthetic response | — | +| `after_request` | After the response is produced — **including on a cache hit**, and on streaming **after** every chunk has been delivered | Observe the completed response, cost, and timing; record and measure | Withhold or rewrite streamed content — the client already has it | +| `on_error` | When a request fails, including a `before_request` rejection or a fail-closed plugin error | Record the failure with the last target attempted | — | + +`on_error` **always records.** It runs on a context detached from the request's cancellation, bounded by a 10-second budget, so a client that disconnected mid-stream still produces a terminal record — the failure never vanishes from the logs. This is why a plugin that logs must list itself at `on_error` too: a failed request never reaches `after_request`. + +## Execution order + +Within a stage, plugins run top-to-bottom in config order. Position matters — a plugin only sees what the plugins above it have already done. + +```yaml +plugins: + # before_request runs top-to-bottom: + - name: word-filter # 1. screen the prompt first + type: guardrail + stage: before_request + enabled: true + config: + blocked_words: ["secret"] + - name: rate-limit # 2. spend a token from the bucket + type: ratelimit + stage: before_request + enabled: true + config: + requests_per_second: 100 + - name: response-cache # 3. a hit sets SkipProvider — steps 1-2 already ran + type: transform + stage: before_request + enabled: true + config: + max_age: 300 + max_entries: 1000 + + # after_request — a multi-stage plugin repeats with BYTE-IDENTICAL config: + - name: response-cache + type: transform + stage: after_request + enabled: true + config: + max_age: 300 + max_entries: 1000 +``` + +Here a cache hit is detected at step 3, so the word filter and the rate limiter have already run — a repeated request is still screened and still spends a token. Listing `response-cache` above the guardrail would not change that: `SkipProvider` skips the provider, not the plugins. + +## Failure policy: a verdict is not a bug + +The gateway distinguishes a plugin that **decided** to deny a request from one that **broke**. + +- **Rejection** — the plugin set `Context.Reject` (with a `Reason`). This is a verdict, honoured for every plugin type, and reaches the client as a client error: `429` for the rate limiter, `402 insufficient_quota` for the budget, a `4xx` for a guardrail. +- **Failure** — the plugin returned an error or panicked (panics are recovered and treated as errors). It never reached a decision, so: + - `guardrail`, `auth`, `ratelimit`, `transform`, and any unknown type **fail closed** — the request aborts with a `500`. A guardrail that could not run has approved nothing; a rate limiter that is down has limited no one, and answering `429` would invite every SDK to retry into the outage. + - `logging` and `metrics` **fail open** — the error is logged and the request proceeds. An observer that dies must not take down the request path. + +To deny a request in your own plugin, set `pctx.Reject` and return `nil`. Return an error only when the plugin itself broke. + +## SkipProvider does not bypass the chain + +A `before_request` plugin can set `Context.SkipProvider` to say "do not call the provider; serve `Context.Response` instead" — this is how `response-cache` answers a hit. It skips the provider call **only**. Every remaining `before_request` plugin and the whole `after_request` stage still run, so a cache hit cannot bypass a guardrail, a rate limit, or a budget listed behind it. `SkipProvider` stays set into `after_request` as a fact — `true` means no provider was contacted — which cost recording keys off and the logger deliberately ignores. + +:::note Removed in v1.4.0 +The old `Context.Skip` (which abandoned every plugin after it, letting a cache hit bypass guardrails) is gone. `SkipProvider` is its replacement. +::: + +## Multi-stage plugins + +A plugin that acts at more than one stage — `response-cache` (check + store), `budget` (check + record), `request-logger` (log at all three) — needs **one config entry per stage**, and the entries must carry **byte-identical config**. The gateway resolves entries by `name` plus the JSON encoding of the whole `config` block: identical entries share one instance (and its state), disagreeing entries are two instances that never see each other. If the entries for one plugin across stages disagree, `ValidateMultiStagePlugins` rejects the config and **the gateway refuses to start** — the same failure `ferrogw validate` and `ferrogw doctor` report. `request-logger` in particular must be listed at `on_error` as well, or failed requests write no terminal row and disappear from the default `/admin/logs` listing. + +## Agentic tool loops + +When a request drives an agentic MCP tool loop, the `before_request` plugins re-run on **every** turn — because each turn is a fresh provider call carrying tool results the caller did not write. Two types are excluded: `transform` (re-running it would rewrite the model mid-conversation) and `logging` / `metrics` (re-running would write one row and one metric sample per turn). So guardrails, the rate limiter, and the budget see every turn — the budget's per-turn check adds this request's running spend, so the cap can close mid-loop — while the logger records the loop once. + +## Secrets in plugin config + +Braced `${VAR}` references inside a plugin's `config` are resolved **at plugin construction**, via the shared env resolver, never at config load. The stored `Config` keeps the reference, so a secret never reaches the config-history store or `GET /admin/config`. A bare `$` is data (`pa$$w0rd` survives); an undefined variable is a startup error. + +## Registering an out-of-tree plugin + +The built-ins have no special path — write your own the same way: + +1. Implement `plugin.Plugin` in your package. +2. Call `plugin.RegisterFactory("my-plugin", New)` from an `init()` function. +3. Add a blank import in `cmd/ferrogw/main.go`: `_ "your/module/path/myplugin"`. + +Optionally implement `ConfigValidator` (`ValidateConfig(map[string]any) error`) so `ferrogw validate` checks your `config` block before deployment. Its contract is narrow — no I/O, no `${VAR}` resolution, no state — because it runs pre-flight on a build machine with no secrets. Deny requests with `pctx.Reject`; reserve returned errors for the plugin actually breaking. + +## Inspecting what is loaded + +Two admin endpoints answer different questions: + +- `GET /admin/plugins` — the plugins **this instance has configured**, read from the live config. +- `GET /admin/plugins/catalog` — the plugins **this build ships**: name, type, one-line summary, the `config` settings each reads, and whether it fails open. This is the authority the dashboard reads, so a card can never name a setting no plugin reads. + +## Built-in plugins + +Six plugins ship with the open-source gateway. Each has its own page for full configuration and gotchas. + +| Plugin | Type | Stages | Purpose | +|---|---|---|---| +| [`word-filter`](/plugins/word-filter) | guardrail | before_request (+ after_request) | Reject a request — or screen a response — whose text contains a blocked entry as a substring. | +| [`max-token`](/plugins/max-token) | guardrail | before_request | Reject a request over a completion-token ceiling, message count, or input length; never imposes a ceiling. | +| [`rate-limit`](/plugins/rate-limit) | ratelimit | before_request | Token-bucket limits globally and per API key or user, independent of the per-IP HTTP limiter. | +| [`budget`](/plugins/budget) | ratelimit | before_request + after_request | Soft per-API-key USD spend cap computed from token usage; `402 insufficient_quota` once exhausted. | +| [`response-cache`](/plugins/response-cache) | transform | before_request + after_request | Serve an identical repeated chat request from an in-memory cache, scoped to the credential that primed it. | +| [`request-logger`](/plugins/request-logger) | logging | before_request + after_request + on_error | Structured per-request logs, optionally persisted to power the dashboard's Request Logs page. | + +Advanced guardrails — PII redaction, prompt-injection shielding, secret scanning, schema validation — are available in Ferro Labs Managed. See [Enterprise plugins](/plugins/enterprise). + +## Related + +- [Configuration](/getting-started/configuration) — where the `plugins:` list lives +- [Rate limiting](/guides/rate-limiting) — the plugin limiter vs the per-IP HTTP limiter +- [Cost tracking](/guides/cost-tracking) — pairing `budget` with request-log cost data +- [Request logging](/operations/request-logging) — persisting `request-logger` output +- [MCP tool calling](/guides/mcp) — how plugins re-run across agentic loop turns +- [Enterprise plugins](/plugins/enterprise) — managed guardrails beyond the OSS set diff --git a/docs/plugins/rate-limit.mdx b/docs/plugins/rate-limit.mdx new file mode 100644 index 0000000..c02eb93 --- /dev/null +++ b/docs/plugins/rate-limit.mdx @@ -0,0 +1,76 @@ +--- +title: Rate Limit Plugin +description: Reference for the rate-limit plugin — global token-bucket rps/burst plus optional per-key and per-user RPM limits, 429 denials, config keys, and YAML examples. +keywords: [rate limit plugin, AI gateway rate limiting, token bucket, requests per second, per-key rate limit, per-user rate limit, 429 too many requests, RATE_LIMIT_RPS] +--- + +The `rate-limit` plugin enforces token-bucket rate limits on gateway traffic: a global requests-per-second bucket, plus optional per-API-key and per-user requests-per-minute buckets layered on top. It runs at `before_request`, rejecting a request with a 429 before it ever reaches a provider. + +This plugin is distinct from the gateway's per-IP HTTP rate limiter, which is a separate, always-on middleware layer configured by the `RATE_LIMIT_RPS` and `RATE_LIMIT_BURST` environment variables — see [Gotchas](#gotchas) for how the two differ. + +## Facts + +| | | +|---|---| +| Stage | `before_request` only | +| Reported type | `ratelimit` (`plugin.TypeRateLimit`) | +| Failure policy | Fails **closed** (a plugin error returns `500`) | +| Denial status | **429**, with a `Retry-After` header | +| Multi-stage | No — a single `plugins[]` entry is enough (unlike `budget`, `response-cache`, `request-logger`) | +| Agentic loops | Re-checked on every MCP tool-loop turn (`RunBeforeLoopTurn`) — each provider call inside a loop spends a token | +| External dependencies | None — in-memory token buckets, per-process. Limits are not shared across replicas | + +## Config keys + +| Key | Type | Default | Required | Description | +|---|---|---|---|---| +| `requests_per_second` | float or int, **must be > 0** | `100` | No | Global request rate applied to all traffic. `0`, negative, `NaN`, and `Inf` are all rejected — both at plugin `Init` and by `ferrogw validate`. A rate of zero would blackhole all traffic rather than disable the limit; use `enabled: false` to turn the plugin off instead. | +| `burst` | float or int, **must be > 0** | `= requests_per_second` | No | Global burst capacity. Same positivity rule as `requests_per_second`. | +| `key_rpm` | float or int, **must be > 0** | unset (per-key limiting off) | No | Per-API-key requests/minute, keyed on `Metadata["api_key"]` (the opaque credential id). Burst equals one full minute's worth of tokens. A request with no key in metadata is not individually limited by this option. Backed by an LRU store capped at 100,000 keys. | +| `user_rpm` | float or int, **must be > 0** | unset (per-user limiting off) | No | Per-user requests/minute, keyed on `Request.User`. A request with an empty `User` field is not individually limited by this option. Same 100,000-entry LRU cap. | + +## Minimal config + +```yaml +plugins: + - name: rate-limit + type: guardrail + stage: before_request + enabled: true + config: + requests_per_second: 100 + burst: 100 + key_rpm: 60 + user_rpm: 30 +``` + +`key_rpm` and `user_rpm` are optional — omit either (or both) to run only the global bucket: + +```yaml +plugins: + - name: rate-limit + type: guardrail + stage: before_request + enabled: true + config: + requests_per_second: 50 + burst: 100 +``` + +## Gotchas + +- **Zero is a load error, not "off."** Every key here is a rate, not a switch: `requests_per_second: 0` (or a negative value, `NaN`, or `Inf`) fails at plugin `Init` and is caught ahead of time by `ferrogw validate` — it never silently starts a gateway that reports healthy and answers 429 to every request forever. To disable the plugin, set `enabled: false`. This is the opposite polarity of `RATE_LIMIT_RPS`, where `0` means no limiting. +- **Not the same limiter as `RATE_LIMIT_RPS`.** The gateway also runs a per-IP HTTP rate limiter as always-on middleware, configured by the `RATE_LIMIT_RPS` / `RATE_LIMIT_BURST` environment variables (default 20 rps / burst 40, enabled unless `RATE_LIMIT_RPS=0`). That limiter keys on the client IP resolved from the trusted-proxy chain (`TRUSTED_PROXIES`) and runs ahead of the plugin pipeline. This plugin keys on the whole gateway process (global bucket) plus, optionally, the API key and the authenticated user — two independent layers that can both reject the same request for different reasons. See [Rate limiting](/guides/rate-limiting) for the full picture of both layers together. +- **Check order: global → per-key → per-user.** The first bucket to deny wins, and the reason in the rejection response says which one it was. All three denials increment the same Prometheus counter, `gateway_rate_limit_rejections_total{key_type="plugin"}` — one label covers all three buckets, so the counter alone cannot tell you which layer is shedding load; read the rejection reason for that. +- **In-memory, per-process.** Token buckets live in process memory. A multi-replica deployment does not share buckets across replicas — each instance enforces its configured rate independently, so the effective fleet-wide rate is roughly `requests_per_second × replica count`. +- **Requests with no key or user skip those buckets, not the request.** A request carrying no `api_key` in metadata (for example under `ALLOW_UNAUTHENTICATED_PROXY=true`) is not individually rate-limited by `key_rpm`; it still passes through the global bucket. Same for `user_rpm` when `Request.User` is empty. +- **Unknown config keys are silently ignored.** `requests_per_minute` (instead of the correct `key_rpm`/`user_rpm`) is not an error — the plugin simply never sees it, and the 100 rps default stays active. Double-check spelling against the table above; there is no config-load error to catch a typo here. +- **Re-checked per agentic loop turn.** Inside an MCP tool loop, `RunBeforeLoopTurn` re-runs this plugin on every turn (unlike `request-logger`, which does not), so a long tool-calling conversation spends one token per provider call, not one per top-level request. +- **Fails closed, on purpose.** If the plugin itself errors — as opposed to denying a request — the request gets a `500`, not a `429`. A broken rate limiter has limited nobody, and answering `429` would invite every retrying client straight into the outage it's supposed to be protecting against. + +## Related + +- [Rate limiting](/guides/rate-limiting) +- [Plugins overview](/plugins) +- [Budget plugin](/plugins/budget) +- [Request logger plugin](/plugins/request-logger) diff --git a/docs/plugins/request-logger.mdx b/docs/plugins/request-logger.mdx new file mode 100644 index 0000000..4b7123a --- /dev/null +++ b/docs/plugins/request-logger.mdx @@ -0,0 +1,88 @@ +--- +title: Request Logger Plugin +description: Reference for the request-logger built-in plugin — structured stdout logs plus optional persisted rows powering the dashboard, config keys, YAML, and gotchas. +keywords: [request logger plugin, AI gateway logging, request log persistence, admin logs, duration_ms, ttft_ms, cost_usd, SQLite Postgres logging] +--- + +The `request-logger` plugin records every request, response, and failure as structured stdout log lines and, when `persist: true` plus a configured request-log store, as rows that power the dashboard's Request Logs page. It runs at all three lifecycle stages — `before_request`, `after_request`, and `on_error` — so a request is logged whether it succeeds, fails, or is served from cache. + +## Facts + +| | | +|---|---| +| Stages | `before_request` **and** `after_request` **and** `on_error` — multi-stage, byte-identical config required at all three | +| Reported type | `logging` (`plugin.TypeLogging`) | +| Failure policy | Fails **open** — a broken log sink costs a warn line, never the request | +| Store write failures | Warned, not returned — a down/full store never fails the request, but is logged so the gap is visible | +| Agentic loops | Does **not** re-run per MCP tool-loop turn (unlike guardrails/ratelimit/budget) — one row per request, not one per turn | +| Timestamps | UTC (`time.Now().UTC()`) on every logged and persisted entry | +| External dependencies | Optional: `REQUEST_LOG_STORE_BACKEND` (`sqlite` or `postgres`) + `REQUEST_LOG_STORE_DSN`, set at the process level — not in plugin `config`. The store is gateway-owned; the plugin's `Close` never touches it | + +## Config keys + +| Key | Type | Default | Required | Description | +|---|---|---|---|---| +| `level` | string: `debug`\|`info`\|`warn`\|`error` | `info` | No | Log level for the `before_request`/`after_request` stdout lines. The `on_error` line always logs at `error`, regardless of this setting. | +| `persist` | bool | `false` | No | Write rows to the shared request-log store. Requires `REQUEST_LOG_STORE_BACKEND` (and `REQUEST_LOG_STORE_DSN`) to be set at the process level; if it isn't, `Init` logs a startup warning and the plugin falls back to stdout-only. | +| `backend` / `dsn` | — | — | No | **Obsolete.** Ignored with a warning at `Init` — persistence target moved to the process-level `REQUEST_LOG_STORE_BACKEND` / `REQUEST_LOG_STORE_DSN` env vars. Deliberately excluded from `GET /admin/plugins/catalog`'s declared settings (only `level` and `persist` are listed). | + +## Minimal config + +All three stage entries must carry byte-identical `config` — the gateway resolves them to one shared instance by `name` + JSON-encoded config, and refuses to start if any of the three disagree. + +```yaml +plugins: + - name: request-logger + type: logging + stage: before_request + enabled: true + config: + level: info + persist: false + + - name: request-logger + type: logging + stage: after_request + enabled: true + config: + level: info + persist: false + + - name: request-logger + type: logging + stage: on_error + enabled: true + config: + level: info + persist: false +``` + +To persist rows for the admin API and dashboard, set `persist: true` on all three entries and configure the store once at the process level: + +```bash +export REQUEST_LOG_STORE_BACKEND=sqlite +export REQUEST_LOG_STORE_DSN=ferrogw-requests.db +``` + +## Gotchas + +- **The `on_error` entry is not optional.** A failed request never reaches `after_request`, so without an `on_error` entry a failure produces no terminal row and simply vanishes from the default `GET /admin/logs` listing — the request happened, but nothing says so. +- **A completed-then-failed request never gets two terminal rows.** If the provider answered, the `after_request` row was written, and a *later* `after_request` plugin then breaks, the failure is annotated onto the row that already exists (`recordLateFailure`) rather than written as a second `on_error` row — otherwise the default listing and every `Stats` figure would double-count that one request. +- **Cache hits are logged like any other request.** Unlike the budget plugin, `request-logger` deliberately ignores `SkipProvider` — a response served from cache still belongs in the audit trail. Its row carries the real token usage from the cached response paired with the `$0` cost the gateway hands a cache hit; usage and cost are different facts, and a cache hit is where they diverge. +- **The `after_request` row's `Provider` and `APIKeyID` answer different questions.** `Provider` names whoever originally produced the response (for a cache hit, the provider that answered the very first time); `APIKeyID` names whoever consumed it *this* time. On a cache hit those can be two different credentials. +- **`on_error` names the provider from the last routing target attempted**, not from a response — there is no response on this path. On a request that never reached routing (denied by a `before_request` plugin, or no target serves the model), that field is empty. +- **Store-write failures are warned about, never returned as plugin errors.** The framework already treats a plugin error as "the plugin itself broke," and a logging plugin is fail-open by design so a struggling store cannot cost a completed LLM call. If `l.writer.Write` fails, the plugin logs `request log write failed; the persisted request log is incomplete` naming the stage, and moves on — the request is unaffected, but the persisted trail is quietly short a row unless you're watching for that warning. +- **`on_error` runs on a context detached from request cancellation**, bounded by a 10-second write budget, specifically so a client that disconnected mid-stream still produces a terminal failure row instead of the record vanishing with the connection. +- **Error text is redacted before it's logged or persisted.** `on_error` runs `pctx.Error.Error()` through the gateway's redactor (emails, JWTs, AWS keys) so an upstream provider error that happened to echo back a credential doesn't land in stdout or the store verbatim. +- **Does not re-run inside agentic MCP tool loops.** Guardrails, the rate limiter, and the budget re-check on every loop turn; `request-logger` is deliberately excluded (along with metrics plugins) so one multi-turn agentic request produces one row, not one row per turn. +- **`GET /admin/logs` defaults to one row per request, not one row per stage.** The plugin writes a row at every stage it runs — `before_request`, `after_request` (or the annotated/on_error terminal row), and `on_error` on failure. The default listing returns only the **terminal** stages (`after_request` + `on_error`) so each request appears once; pass `?stage=all` to see the raw per-stage event stream, or `?stage=` to filter to exactly one stage. +- **`backend`/`dsn` in plugin config are a no-op**, kept only so an operator running an old config gets a warning explaining where the setting moved, rather than a silently ignored key. Set `REQUEST_LOG_STORE_BACKEND`/`REQUEST_LOG_STORE_DSN` instead. +- **Unknown config keys are silently ignored** — the plugin reads its own map rather than going through the strict top-level config decoder, so a typo like `persists: true` leaves `persist` at its `false` default with no error. + +## Related + +- [Request logging](/operations/request-logging) — persisting rows, store backends, and admin log filters +- [Dashboard](/guides/dashboard) — the embedded Request Logs page this plugin's persisted rows power +- [Plugins overview](/plugins) +- [Budget plugin](/plugins/budget) +- [Response cache plugin](/plugins/response-cache) diff --git a/docs/plugins/response-cache.mdx b/docs/plugins/response-cache.mdx new file mode 100644 index 0000000..0b8b1b8 --- /dev/null +++ b/docs/plugins/response-cache.mdx @@ -0,0 +1,66 @@ +--- +title: Response Cache Plugin +description: Reference for the response-cache plugin — in-memory LRU+TTL caching of identical chat requests scoped per credential, with config keys, YAML, and gotchas. +keywords: [response cache, AI gateway plugin, LLM response caching, cache hit, SkipProvider, ModelPreserving, in-memory cache] +--- + +The `response-cache` plugin serves an identical repeated chat request from an in-memory LRU+TTL store instead of calling the provider, cutting cost and latency on repeated prompts. It runs at both `before_request` (serve a hit) and `after_request` (store a miss), and every cache entry is scoped to the API credential that primed it. + +## Facts + +| | | +|---|---| +| Stages | `before_request` **and** `after_request` — multi-stage, byte-identical config required at both | +| Reported type | `transform` | +| Failure policy | Fails **closed** (a plugin error returns `500`) | +| `ModelPreserving` | Yes — declaring it does **not** disable the gateway's pre-plugin `admitModel` check | +| External dependencies | None — in-memory (`pkg/cache.Memory`), per-process, not shared across replicas | +| Surfaces covered | Chat only — stands down on embeddings/images (any request carrying `Metadata[surface]`) | + +## Config keys + +| Key | Type | Default | Required | Description | +|---|---|---|---|---| +| `max_age` | int (seconds) | `300` | No | TTL for a cached entry. | +| `max_entries` | int | `1000` | No | LRU capacity. `<= 0` disables storing — `after_request` becomes a no-op (cache reads still run, but nothing new is ever written). | + +## Minimal config + +Both stage entries must carry byte-identical `config` — the gateway resolves them to one shared cache instance by `name` + JSON-encoded config, and refuses to start if the two disagree. + +```yaml +plugins: + - name: response-cache + type: transform + stage: before_request + enabled: true + config: + max_age: 300 + max_entries: 1000 + + - name: response-cache + type: transform + stage: after_request + enabled: true + config: + max_age: 300 + max_entries: 1000 +``` + +## Gotchas + +- **Entries are scoped to the credential that primed them, with no opt-out.** The cache key is a SHA-256 hash over the opaque API key ID plus the request fields, so one credential's response is never served to another. There is no config flag to share a cache across keys — running many API keys against identical prompts lowers the effective hit rate, because each key builds its own set of entries. Unauthenticated callers share one bucket among themselves, never with an authenticated caller. +- **A cache hit only skips the provider call.** `SkipProvider` (not the removed `Context.Skip`) suppresses calling the provider — every remaining `before_request` plugin behind response-cache (guardrails, rate limiting, budget) still runs, and the entire `after_request` stage still runs, including budget's recording step and request-logger. A hit cannot be used to bypass a guardrail or a budget check. Plugins listed *before* response-cache in the config run before the hit is even detected. +- **What's in the key.** The hash covers `model`, every message (`role`, `name`, `content`, content parts including image URLs and detail, tool calls with function name/arguments, tool call id, reasoning content), plus `temperature`, `top_p`, `n`, `seed`, `max_tokens`/`max_completion_tokens`, penalties, `stop`, `tools`, `tool_choice`, `parallel_tool_calls`, `response_format`, `logprobs`/`top_logprobs`, `stream`, `user`, and `logit_bias`. The provider/target is deliberately **not** in the key — a model id is assumed to name the same model on every target that serves it. +- **Chat-only.** It stands down entirely on embeddings and image requests (any request carrying `Metadata[surface]`), because those are internally projected onto the chat request shape for routing and would hash identically to a chat request asking the same text — while the response has no `Choices` to serve back. +- **`ModelPreserving` matters.** Because response-cache implements `plugin.ModelPreserving`, configuring it does not turn off the gateway's pre-plugin `admitModel` check the way an ordinary `transform` plugin would. Without that interface, an unroutable model could spend a rate-limit token and a budget dollar on its way to a 404. +- **Cached responses cost nothing and are still logged.** A served hit reports `$0` cost but is still recorded by request-logger and still counted by rate-limit/budget — it just adds no spend. +- **In-memory, per-process.** There is no Redis backend; a multi-replica deployment gets independent caches with independently lower hit rates, and nothing survives a restart. +- **Re-storing a hit is skipped.** `after_request` is a no-op when `SkipProvider` is set or `Response` is nil, so serving a cached response never refreshes its TTL from that request. + +## Related + +- [Plugins overview](/plugins) +- [Budget plugin](/plugins/budget) +- [Request logger plugin](/plugins/request-logger) +- [Request lifecycle](/getting-started/request-lifecycle) diff --git a/docs/plugins/word-filter.mdx b/docs/plugins/word-filter.mdx new file mode 100644 index 0000000..6c49e81 --- /dev/null +++ b/docs/plugins/word-filter.mdx @@ -0,0 +1,80 @@ +--- +title: Word Filter plugin +description: Reference for the word-filter guardrail plugin — substring blocklist matching on request and response text, config keys, YAML example, and edge cases. +keywords: [word-filter, content guardrail, blocklist, substring match, AI gateway plugin, before_request, after_request] +--- + +`word-filter` is a guardrail plugin that rejects a request whose message text contains a configured entry as a **substring** — deliberately not word-boundary-aware. Listed only at `before_request` it screens the incoming request; listed at `after_request` too, it also screens the response's choices. + +## Facts + +| | | +|---|---| +| Stage(s) | `before_request` (screens request messages); optionally also `after_request` (screens response choices) | +| Reported type | `guardrail` ([`plugin.TypeGuardrail`](/plugins)) | +| Failure policy | Fails **closed** — a plugin error (not a rejection) aborts the request as a `500` | +| Multi-stage | No — a single `before_request` entry is a complete, valid configuration. Add an `after_request` entry only to also screen responses; the two are independent, not a matched pair like `response-cache`/`budget` | +| External dependencies | None — in-memory only | +| `ValidateConfig` | Not implemented — `ferrogw validate` does not catch a malformed config block for this plugin | + +On `before_request` a rejection reaches the caller as `400 invalid_request_error` (`request_rejected`). On `after_request` a rejection reaches the caller as `502 upstream_error` (`response_rejected`) — the stage runs after the provider has already answered, so a match there is reported, not withheld. + +## Configuration + +| Key | Type | Default | Required | Description | +|---|---|---|---|---| +| `blocked_words` | list of strings | `[]` (no-op when empty) | No | Entries matched as substrings against each message's `Content` and every `ContentPart.Text`. Non-string list items are silently dropped. | +| `case_sensitive` | bool | `false` | No | When `false`, the blocklist is lowercased once at `Init` and content is lowercased per check. | + +Unknown keys in the `config` block are silently ignored. + +## Minimal config + +```yaml +plugins: + - name: word-filter + type: guardrail + stage: before_request + enabled: true + config: + blocked_words: ["password", "secret"] + case_sensitive: false +``` + +To also screen the model's response, add a second, independent entry at `after_request`: + +```yaml +plugins: + - name: word-filter + type: guardrail + stage: before_request + enabled: true + config: + blocked_words: ["password", "secret"] + case_sensitive: false + + - name: word-filter + type: guardrail + stage: after_request + enabled: true + config: + blocked_words: ["password", "secret"] + case_sensitive: false +``` + +## Gotchas + +- **Substring, not word-boundary matching.** `"ass"` blocks `"class"`. This is deliberate: a boundary-aware matcher is evaded by punctuation, concatenation, and zero-width characters, and each miss is a prompt that reached the provider. There is no whole-word mode. +- **Uninspectable content is rejected, not passed through.** An embeddings request whose input arrives as token IDs (rather than text) cannot be screened. Rather than fail open, `word-filter` rejects it at `before_request` with a generic content-policy reason — this only applies when at least one `blocked_words` entry is configured, and only at `before_request` (an unreadable *request* says nothing about what an `after_request` check should do with the *response*, which has already been returned by the provider). +- **The matched word is never returned to the client** — only logged server-side (`word-filter: blocked request`/`response`, field `matched_word`). The rejection reason sent to the caller is a fixed string (`"request blocked by content policy"` / `"response blocked by content policy"`). This is blocklist-probe protection: an attacker cannot binary-search your blocklist through the API. +- **Image URLs and data URIs are not screened.** `ContentPart.ImageURL` is deliberately skipped — a base64-encoded image contains any given three-letter word by chance, so treating it as text would produce constant false positives. Screening image content is an OCR problem, not a string-matching one. +- **Both `Content` and `ContentParts[].Text` are checked.** A message decoded from JSON collapses only `"text"`-typed parts into `Content`; a part of any other type (e.g. a custom `input_text`) leaves no trace there. Both are scanned so neither path is a bypass. +- **Streaming responses can't be un-served.** On a streaming completion, the `after_request` stage runs only after chunks have already been delivered to the caller — a match is logged, but the content already reached the client. Put the guardrail at `before_request` (screening the prompt) if withholding matters, or avoid streaming for that route. +- **No `ValidateConfig`.** Unlike `rate-limit`, a malformed `blocked_words` value (e.g. a plain string instead of a list) is silently accepted with an empty effective blocklist — `ferrogw validate` will not flag it. +- **Plugins are global, not per-route.** There is no per-target or per-route way to scope a blocklist; every request through the gateway is screened by every enabled `word-filter` entry. + +## Related + +- [Plugins overview](/plugins) +- [Max token guardrail](/plugins/max-token) +- [Getting started: configuration](/getting-started/configuration) diff --git a/docs/providers/configuration.mdx b/docs/providers/configuration.mdx new file mode 100644 index 0000000..ba606b8 --- /dev/null +++ b/docs/providers/configuration.mdx @@ -0,0 +1,254 @@ +--- +title: Provider configuration +description: "Configure all 30 AI Gateway providers via environment variables: API keys, base-URL overrides, and special config for Azure, Bedrock, Vertex AI, and Ollama." +keywords: [provider configuration, API key environment variables, provider base URL, Azure OpenAI setup, AWS Bedrock credentials, Vertex AI auth, Ollama configuration, Cloudflare Workers AI, Databricks, Replicate] +--- + +Provider credentials are supplied through **environment variables** and never live in `config.yaml`. A config target only *names* a provider by its `virtual_key`; the credential that activates that provider comes from the environment: + +```yaml +targets: + - virtual_key: openai # names the provider — credentials come from OPENAI_API_KEY +``` + +A provider is **auto-registered when its required environment variables are present** at startup. No provider entry needs editing in code and no key is ever written to disk in the config. + +:::info Registration is not routing +Setting a provider's env vars registers it, but it only serves traffic if a `targets[]` entry names its `virtual_key`. A request for a model no configured target owns returns **404 `model_not_found`** even when the provider is registered and healthy. +::: + +:::info Credentials are enforced at construction (v1.4.0) +The required-key gate runs whenever a provider is built — whether credentials come from environment variables or from a programmatic config map. A provider is never constructed without its credential: on the environment path a missing required var **skips** the provider silently; on the programmatic path it is an **error**. +::: + +Provider credentials are read directly from their own variables. Elsewhere in config — `plugins[].config`, `mcp_servers[].headers` / `env`, and `observability.exporters[].config` — you can reference an environment variable with `${VAR}`. Only the braced form `${NAME}` is a reference (a bare `$` is literal data, so `pa$$w0rd` survives byte-for-byte). References resolve **when the component is constructed**, not at config load, so a secret never reaches the config-history store or `GET /admin/config`; an undefined variable is a startup error that names every missing variable. + +## The base-URL rule {#base-url-rule} + +Since v1.4.0, `_BASE_URL` is the **API root, used verbatim** — version segment included. Each surface (chat, streaming, embeddings, images, model discovery, `/v1/*` pass-through) appends only its operation path to that root, and the root is resolved **once**, when the provider is constructed. Write it exactly as the vendor documents it: + +```bash +# Reaches https://proxy.example.com/v1/chat/completions and .../v1/embeddings +export OPENAI_BASE_URL=https://proxy.example.com/v1 +``` + +- **Include the version segment.** Write `https://api.groq.com/openai/v1`, not `.../openai`. +- A base with **no path at all** is the one case with a safety net: it resolves to the provider's own default version segment (`/v1` for OpenAI-wire providers, `/v1beta` for Gemini). So `http://host:9901` and `http://host:9901/v1` are equivalent. A base that carries *any* path is taken as written. +- Userinfo travels with the host: `https://user:pass@proxy.example.com` reaches the proxy authenticated. +- A query string or fragment is **refused at startup** — an operation path is appended to the root, so a query would bury the operation. + +**Host-root exceptions.** Three providers are configured with a host rather than an API root, because the value genuinely is not one: + +| Provider | Variable | Write it as | +|---|---|---| +| Cohere | `COHERE_BASE_URL` | the host — `https://api.cohere.com` (chat is `/v2/chat`, embeddings `/v1/embed`, so no single API root exists) | +| Ollama | `OLLAMA_HOST` | the server root — `http://localhost:11434` (OpenAI surface at `/v1`, native API at `/api`) | +| Azure AI Foundry | `AZURE_FOUNDRY_ENDPOINT` | the resource host — `https://.services.ai.azure.com` (the gateway appends Azure's fixed `/openai/v1`) | + +`DATABRICKS_HOST`, `AZURE_OPENAI_ENDPOINT`, and `HUGGING_FACE_ENDPOINT` are resource hosts too — not `_BASE_URL` overrides. The provider builds the vendor's fixed surface path beneath them. + +## OpenAI-compatible providers {#openai-compatible} + +These providers register with a single API key and accept an optional `_BASE_URL` override (see [the base-URL rule](#base-url-rule) above): + +```bash +export OPENAI_API_KEY=sk-... +export GROQ_API_KEY=gsk_... +export DEEPSEEK_API_KEY=... +# ...and so on for any provider below +``` + +| Provider | API key env var | Base-URL override | Default API root | +|---|---|---|---| +| AI21 Labs | `AI21_API_KEY` | `AI21_BASE_URL` | `https://api.ai21.com/studio/v1` | +| Cerebras | `CEREBRAS_API_KEY` | `CEREBRAS_BASE_URL` | `https://api.cerebras.ai/v1` | +| DeepInfra | `DEEPINFRA_API_KEY` | `DEEPINFRA_BASE_URL` | `https://api.deepinfra.com/v1/openai` | +| DeepSeek | `DEEPSEEK_API_KEY` | `DEEPSEEK_BASE_URL` | `https://api.deepseek.com/v1` | +| Fireworks AI | `FIREWORKS_API_KEY` | `FIREWORKS_BASE_URL` | `https://api.fireworks.ai/inference/v1` | +| Groq | `GROQ_API_KEY` | `GROQ_BASE_URL` | `https://api.groq.com/openai/v1` | +| Mistral AI | `MISTRAL_API_KEY` | `MISTRAL_BASE_URL` | `https://api.mistral.ai/v1` | +| Moonshot AI | `MOONSHOT_API_KEY` | `MOONSHOT_BASE_URL` | `https://api.moonshot.ai/v1` | +| Novita | `NOVITA_API_KEY` | `NOVITA_BASE_URL` | `https://api.novita.ai/openai/v1` | +| NVIDIA NIM | `NVIDIA_NIM_API_KEY` | `NVIDIA_NIM_BASE_URL` | `https://integrate.api.nvidia.com/v1` | +| OpenAI | `OPENAI_API_KEY` | `OPENAI_BASE_URL` | `https://api.openai.com/v1` | +| OpenRouter | `OPENROUTER_API_KEY` | `OPENROUTER_BASE_URL` | `https://openrouter.ai/api/v1` | +| Perplexity | `PERPLEXITY_API_KEY` | `PERPLEXITY_BASE_URL` | `https://api.perplexity.ai` | +| Qwen (DashScope) | `QWEN_API_KEY` | `QWEN_BASE_URL` | `https://dashscope-intl.aliyuncs.com/compatible-mode/v1` | +| SambaNova | `SAMBANOVA_API_KEY` | `SAMBANOVA_BASE_URL` | `https://api.sambanova.ai/v1` | +| Together AI | `TOGETHER_API_KEY` | `TOGETHER_BASE_URL` | `https://api.together.ai/v1` | +| xAI (Grok) | `XAI_API_KEY` | `XAI_BASE_URL` | `https://api.x.ai/v1` | + +## Anthropic {#anthropic} + +A single API key. `ANTHROPIC_BASE_URL` is an optional verbatim API-root override (default `https://api.anthropic.com/v1`). Anthropic uses its native Messages wire, so `/v1/*` pass-through proxying returns **501** by design. + +```bash +export ANTHROPIC_API_KEY=sk-ant-... +``` + +## Google Gemini {#gemini} + +A single API key. `GEMINI_BASE_URL` is an optional verbatim override; a path-less base resolves to `/v1beta` (the provider's default). Gemini uses its native `generateContent` wire, so `/v1/*` pass-through returns **501**. + +```bash +export GEMINI_API_KEY=... +``` + +## Cohere {#cohere} + +A single API key. `COHERE_BASE_URL` is a **host-root** override — write only the host (see [host-root exceptions](#base-url-rule)). Cohere uses its native v2 wire; the gateway's `/v1/rerank` endpoint follows the Cohere-v2 contract, and `/v1/*` pass-through returns **501**. + +```bash +export COHERE_API_KEY=... +export COHERE_BASE_URL=https://api.cohere.com # optional; host only +``` + +## Hugging Face {#hugging-face} + +A single API key. The override variable is `HUGGING_FACE_ENDPOINT` (config key `base_url`), **not** `HUGGING_FACE_BASE_URL` — it points at a dedicated Inference Endpoint, a resource host. Default root: `https://router.huggingface.co/v1`. + +```bash +export HUGGING_FACE_API_KEY=hf_... +export HUGGING_FACE_ENDPOINT=https://router.huggingface.co/v1 # optional +``` + +## Azure OpenAI {#azure-openai} + +Requires the resource endpoint and a deployment name in addition to the key. The API key is sent as the `api-key` header. The deployment name **is** the model set — this is the only provider with a required `deployment` config key. + +```bash +export AZURE_OPENAI_API_KEY=... +export AZURE_OPENAI_ENDPOINT=https://.openai.azure.com # resource host +export AZURE_OPENAI_DEPLOYMENT=gpt-4o # required +export AZURE_OPENAI_API_VERSION=2024-10-21 # optional; default 2024-10-21 +``` + +Request URLs are built as `{endpoint}/openai/deployments/{deployment}/{op}?api-version=...`; the batch surface uses the resource's `/openai/v1` root. Pass-through proxying returns **501** (native wire). + +## Azure AI Foundry {#azure-foundry} + +Requires the resource host. The endpoint is a **host**, not an API root — the gateway appends Azure's fixed GA `/openai/v1` route. + +```bash +export AZURE_FOUNDRY_API_KEY=... +export AZURE_FOUNDRY_ENDPOINT=https://.services.ai.azure.com # resource host +export AZURE_FOUNDRY_API_VERSION=... # optional +``` + +## AWS Bedrock {#bedrock} + +Bedrock is built on `aws-sdk-go-v2`, so it has no HTTP base URL. It is considered configured when **any** of `AWS_REGION`, `AWS_ACCESS_KEY_ID`, or `AWS_BEARER_TOKEN_BEDROCK` is set — three alternative credential modes. Region defaults to `us-east-1`. + +```bash +# 1. API-key (bearer) auth +export AWS_BEARER_TOKEN_BEDROCK=... +export AWS_REGION=us-east-1 # optional; defaults to us-east-1 + +# 2. Static credentials (SigV4) +export AWS_ACCESS_KEY_ID=AKIA... +export AWS_SECRET_ACCESS_KEY=... +export AWS_SESSION_TOKEN=... # optional +export AWS_REGION=us-east-1 + +# 3. Instance role / credential chain +export AWS_REGION=us-east-1 +``` + +Bedrock uses its native `InvokeModel` API; pass-through proxying returns **501**. + +## Google Vertex AI {#vertex-ai} + +Requires the project ID (the activation gate) and region — `VERTEX_AI_REGION` is required once `VERTEX_AI_PROJECT_ID` is set, or construction errors. Authentication resolves through one of three paths: + +```bash +export VERTEX_AI_PROJECT_ID=my-gcp-project +export VERTEX_AI_REGION=us-central1 + +# then one of: +export VERTEX_AI_API_KEY=... +# or a service-account JSON string or path (JWT exchanged for the cloud-platform scope): +export VERTEX_AI_SERVICE_ACCOUNT_JSON='{"type":"service_account",...}' +# or leave both unset to use Application Default Credentials / workload identity +``` + +The base URL is built from config as `https://{region}-aiplatform.googleapis.com/v1/projects/{project_id}/locations/{region}/endpoints/openapi`. Native wire; pass-through returns **501**. + +## Cloudflare Workers AI {#cloudflare} + +The only provider that requires an account ID. `CLOUDFLARE_BASE_URL` is an optional override; the default root interpolates the account ID. + +```bash +export CLOUDFLARE_API_KEY=... +export CLOUDFLARE_ACCOUNT_ID=... +``` + +Default API root: `https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/v1`. + +## Databricks {#databricks} + +Requires the workspace host and a token. `DATABRICKS_TOKEN` maps to config key `api_key`; `DATABRICKS_HOST` is a resource host (config key `base_url`) from which the gateway builds the vendor's fixed surface path. + +```bash +export DATABRICKS_TOKEN=dapi... +export DATABRICKS_HOST=https://.cloud.databricks.com # workspace host +``` + +## Replicate {#replicate} + +The model set comes from config, so declare the models you intend to serve. Optional `REPLICATE_TEXT_MODELS` / `REPLICATE_IMAGE_MODELS` are comma-separated lists (config keys `text_models` / `image_models`). + +```bash +export REPLICATE_API_TOKEN=r8_... +export REPLICATE_TEXT_MODELS=meta/llama-3-8b-instruct,mistralai/mistral-7b-instruct-v0.2 +export REPLICATE_IMAGE_MODELS=black-forest-labs/flux-schnell +``` + +:::note +Replicate authenticates with `REPLICATE_API_TOKEN` (not `_API_KEY`); its config key is `api_token`, not `api_key`. Native wire — pass-through returns **501**. +::: + +## Ollama {#ollama} + +Local Ollama has **no API key**. `OLLAMA_HOST` (config key `host`) is both the activation gate and a **server root**, not an API root (see [host-root exceptions](#base-url-rule)). Ollama serves any model it has pulled, so `FERRO_OLLAMA_MODELS` only narrows what `/v1/models` advertises. + +```bash +export OLLAMA_HOST=http://localhost:11434 +export FERRO_OLLAMA_MODELS=llama3.2,gpt-oss:20b,mistral # optional; narrows /v1/models +``` + +:::warning Set the Ollama model list with `FERRO_OLLAMA_MODELS` +`OLLAMA_MODELS` is Ollama's **own** variable for its models *directory* (`$HOME/.ollama/models`), so it must not be used to list models here — it is deprecated in this gateway and read for one more release with a startup `WARN`, and a path-shaped value (no comma) is dropped with a `WARN`. When both are set, `FERRO_OLLAMA_MODELS` wins. +::: + +## Ollama Cloud {#ollama-cloud} + +The hosted Ollama service, which does require an API key. `OLLAMA_CLOUD_BASE_URL` (default `https://ollama.com/v1`) and `OLLAMA_CLOUD_MODELS` are optional. + +```bash +export OLLAMA_API_KEY=... +export OLLAMA_CLOUD_MODELS=gpt-oss:20b # optional; comma-separated +``` + +Ollama Cloud is the only provider with no `/v1/*` pass-through surface. + +## Verify configured providers {#verify} + +After starting the gateway, confirm which providers registered: + +```bash +# Unauthenticated: per-provider status, model counts, and circuit state +curl http://localhost:8080/health + +# Registered models grouped by provider (needs a bearer token — an API key or MASTER_KEY) +curl -H "Authorization: Bearer $MASTER_KEY" http://localhost:8080/v1/models +``` + +`/health` stays unauthenticated and reports provider names, model counts, and circuit state. `/v1/models` and the rest of `/v1/*` require a bearer token unless `ALLOW_UNAUTHENTICATED_PROXY=true`. + +## Related + +- [Providers overview](/providers) — endpoint matrix and supported surfaces per provider +- [Configuration reference](/getting-started/configuration) — the full config schema +- [Server settings](/operations/server-settings) — env vars, timeouts, and limits +- [Authentication](/guides/auth) — bearer tokens, scopes, and `MASTER_KEY` +- [Routing](/routing) — targets, strategies, and failover diff --git a/docs/providers/overview.mdx b/docs/providers/overview.mdx new file mode 100644 index 0000000..d8826c2 --- /dev/null +++ b/docs/providers/overview.mdx @@ -0,0 +1,167 @@ +--- +title: Providers +description: "One OpenAI-compatible API in front of 30 LLM providers: the full provider list, the endpoint-support matrix, and how each provider is declared and activated." +keywords: [ai gateway providers, openai-compatible api, llm providers, provider endpoint matrix, ferro labs, openai anthropic gemini, provider configuration] +slug: /providers +--- + +The Ferro Labs AI Gateway speaks to **30 LLM providers** behind a single +OpenAI-compatible API. Point any client written against the OpenAI SDK at the +gateway and it reaches every provider unchanged — the gateway translates each +request and response to and from the OpenAI shape, whether the upstream is an +OpenAI-compatible surface or a native wire (Anthropic Messages, Gemini +`generateContent`, Cohere v2, Bedrock InvokeModel, Vertex, Replicate). + +This page is the provider directory: the logo grid links each provider to its +per-provider setup, and the [endpoint-support matrix](#endpoint-support-matrix) +shows which surfaces each one implements. For the catalogue of individual models +(2,500+ across all providers), browse **[ferrolabs.ai/models](https://www.ferrolabs.ai/models)** — this +page does not list models. + +:::warning `targets` is an allowlist (v1.4.0+) +Setting a provider's environment variable **registers** it, but registration is +not routing. A provider only serves traffic when it is listed under `targets[]` +in your config. A request for a model owned only by an unlisted provider returns +**404 `model_not_found`**, even though the provider is present in the process and +healthy. Registration is driven by which credentials the environment holds; +routing is driven by which names the config lists — the two sets can be disjoint. +See [Routing](/routing) and [Provider configuration](/providers/configuration). +::: + +## Supported providers + + + +## Endpoint-support matrix + +Which OpenAI-compatible surface each provider implements, as of **v1.4.1**. This +mirrors the machine-checked matrix in the gateway source (`providers/README.md`), +which fails the build if a provider's real interface set ever drifts from it. + +- **✓** — a typed surface (`Complete`, `Embed`, `Rerank`, …), or a working + `/v1/*` pass-through for the **Proxy** column. +- **—** — not implemented, or (for **Proxy**) a native-wire provider whose raw + `/v1/*` pass-through returns **501** by design. Reach those through the + translated surfaces to their left. + +**Audio** groups transcription (STT) and speech (TTS) — ✓ if a provider serves +either. **Rerank** uses the Cohere-v2 contract; every other column uses the +OpenAI contract. Every provider implements **Chat** and **Stream** by definition. + +
+ +| Provider | Chat | Stream | Embed | Image | Audio | Rerank | Moder. | Batch | Responses | Proxy | +|---|:--:|:--:|:--:|:--:|:--:|:--:|:--:|:--:|:--:|:--:| +| AI21 Labs | ✓ | ✓ | — | — | — | — | — | — | — | ✓ | +| Anthropic | ✓ | ✓ | — | — | — | — | — | — | — | — | +| Azure AI Foundry | ✓ | ✓ | ✓ | — | — | — | — | — | — | — | +| Azure OpenAI | ✓ | ✓ | ✓ | ✓ | ✓ | — | — | ✓ | — | — | +| AWS Bedrock | ✓ | ✓ | ✓ | ✓ | — | ✓ | — | — | — | — | +| Cerebras | ✓ | ✓ | — | — | — | — | — | — | — | ✓ | +| Cloudflare Workers AI | ✓ | ✓ | ✓ | — | — | — | — | — | — | ✓ | +| Cohere | ✓ | ✓ | ✓ | — | — | ✓ | — | — | — | — | +| Databricks | ✓ | ✓ | ✓ | — | — | — | — | — | — | ✓ | +| DeepInfra | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | — | — | — | ✓ | +| DeepSeek | ✓ | ✓ | — | — | — | — | — | — | — | ✓ | +| Fireworks AI | ✓ | ✓ | ✓ | — | ✓ | — | — | — | — | ✓ | +| Google Gemini | ✓ | ✓ | ✓ | ✓ | — | — | — | — | — | — | +| Groq | ✓ | ✓ | — | — | ✓ | — | — | ✓ | — | ✓ | +| Hugging Face | ✓ | ✓ | ✓ | ✓ | — | — | — | — | — | ✓ | +| Mistral AI | ✓ | ✓ | ✓ | — | ✓ | — | ✓ | — | — | ✓ | +| Moonshot AI | ✓ | ✓ | — | — | — | — | — | — | — | ✓ | +| Novita | ✓ | ✓ | ✓ | — | — | — | — | ✓ | — | ✓ | +| NVIDIA NIM | ✓ | ✓ | ✓ | — | — | ✓ | — | — | — | ✓ | +| Ollama | ✓ | ✓ | ✓ | — | — | — | — | — | — | ✓ | +| Ollama Cloud | ✓ | ✓ | ✓ | — | — | — | — | — | — | — | +| OpenAI | ✓ | ✓ | ✓ | ✓ | ✓ | — | ✓ | ✓ | ✓ | ✓ | +| OpenRouter | ✓ | ✓ | ✓ | — | — | — | — | — | — | ✓ | +| Perplexity | ✓ | ✓ | — | — | — | — | — | — | — | ✓ | +| Qwen | ✓ | ✓ | ✓ | — | — | — | — | ✓ | — | ✓ | +| Replicate | ✓ | ✓ | — | ✓ | — | — | — | — | — | — | +| SambaNova | ✓ | ✓ | ✓ | — | ✓ | — | — | — | — | ✓ | +| Together AI | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | — | — | — | ✓ | +| Google Vertex AI | ✓ | ✓ | ✓ | ✓ | — | — | — | — | — | — | +| xAI | ✓ | ✓ | — | ✓ | — | — | — | — | ✓ | ✓ | + +
+ +**Totals** — Chat / Stream: 30 · Embeddings: 21 · Images: 10 · Audio: 9 · Rerank: +5 (`bedrock`, `cohere`, `deepinfra`, `nvidia-nim`, `together`) · Moderations: 2 +(`openai`, `mistral`) · Batch: 5 (`openai`, `azure-openai`, `groq`, `novita`, +`qwen`) · Responses: 2 (`openai`, `xai`) · Proxy pass-through: 21. + +Nine providers show **—** for **Proxy**. Eight are native-wire — Anthropic, AWS +Bedrock, Cohere, Google Gemini, Google Vertex AI, Replicate, and the two Azure +surfaces — whose raw `/v1/*` pass-through returns **501** by design (reach them +through the translated columns). Ollama Cloud is the one provider that is not +proxiable at all. A **—** in a capability column is not a routing limitation: +those providers still route every model the catalog, live discovery, or +`targets[].models` names. + +## How a provider is declared + +A provider has two independent parts: **credentials** (from the environment) and +a **target** (in your config). Both must be present for the provider to serve. + +**Credentials never live in `config.yaml`.** Each provider reads its API key and +any special settings from environment variables — `OPENAI_API_KEY`, +`ANTHROPIC_API_KEY`, `AZURE_OPENAI_ENDPOINT`, and so on. A provider is +auto-registered the moment its required environment variables are present; no +`main.go` edit and no config entry is needed to register it. + +**A target names a provider by `virtual_key`.** In config, a routing target +refers to a registered provider by its id — that id *is* the whole declaration: + +```yaml +strategy: + mode: fallback +targets: + - virtual_key: openai # the provider id — this is the declaration + retry: + attempts: 3 + - virtual_key: anthropic +``` + +Only providers listed under `targets[]` route traffic (see the allowlist warning +above). For every provider's exact environment variables, optional +`_BASE_URL` override, and any special configuration keys, see +**[Provider configuration](/providers/configuration)**. + +## Related + +- [Provider configuration](/providers/configuration) — per-provider environment variables and special settings +- [Routing](/routing) — how `targets[]`, strategies, and failover work +- [Model catalogue](https://www.ferrolabs.ai/models) — every model across all 30 providers +- [Quickstart](/getting-started/quickstart) — send your first request through the gateway +- [API reference](/api-reference/overview) — the OpenAI-compatible endpoints each column maps to diff --git a/docs/routing/ab-test.mdx b/docs/routing/ab-test.mdx new file mode 100644 index 0000000..9e9cb9d --- /dev/null +++ b/docs/routing/ab-test.mdx @@ -0,0 +1,65 @@ +--- +title: A/B test routing strategy +description: "Configure strategy.mode: ab-test in Ferro Labs AI Gateway to weight-split live traffic across labelled variants for model migration or cost comparison." +keywords: [ab test routing, a/b test, weighted variants, model migration, ai-gateway strategy.mode, ab_variants, traffic split] +--- + +A/B test splits live traffic across two or more labelled variants by weighted random draw. It optimizes for **comparing** models or providers under real production traffic — measuring quality, cost, or latency differences during a migration — not for failover order (`fallback`), even load distribution (`loadbalance`), or measured speed (`least-latency`). Every request is answered by one real provider: this is **not** a shadow-traffic or mirroring mode. Set `strategy.mode: ab-test` to use it. + +## Behaviour + +`ab-test` is a **pool mode**: on a request failure, the pipeline advances to the next configured target rather than reporting the failure back to the caller — but only after the strategy has already committed to a drawn variant (see [Gotchas](#gotchas)). + +`ABTest.SelectTargets` (`internal/strategies/abtest.go`) filters `ab_variants[]` down to those whose provider is registered and whose `SupportsModel(req.Model)` returns true, then makes one weighted random draw (`weightedPick`) over that eligible subset. The drawn variant's `target_key` leads the returned order, followed by every configured `targets[].virtual_key` (de-duplicated) as pipeline fallback candidates. `SelectTargets` returns `nil` — the caller reports `404 model_not_found` — when no variant is both eligible for the requested model and positively weighted. + +## Config keys + +| Key | Type | Default | Description | +|---|---|---|---| +| `strategy.mode` | string | — | Set to `ab-test` (hyphen). | +| `strategy.ab_variants[].target_key` | string | — | Must name one of the configured `targets[].virtual_key`. | +| `strategy.ab_variants[].weight` | float64 | — | Relative traffic share = `weight / sum(weights)`. `0` drains the variant (no traffic); negative or an all-zero set is rejected at load. | +| `strategy.ab_variants[].label` | string | — | Human-readable variant id (e.g. `control`, `challenger`), logged with every routed request. | + +## Minimal working YAML + +```yaml +strategy: + mode: ab-test + ab_variants: + - target_key: openai + weight: 70 + label: control + - target_key: anthropic + weight: 30 + label: challenger + +targets: + - virtual_key: openai + - virtual_key: anthropic +``` + +## When to use + +- Comparing model quality or cost across providers under real production traffic before committing fully. +- A gradual migration: shift weights toward the challenger over successive deploys, then remove the control once satisfied. +- Draining a variant to zero traffic (`weight: 0`) before removing it from the config, the same pattern `loadbalance` uses for targets. + +## Gotchas + +- **Not shadow traffic.** Every request is routed to exactly one real provider — the drawn variant. There is no mirrored or duplicated call to compare responses side by side; comparison happens by analyzing labelled logs after the fact. +- **The draw is over eligible variants, not all configured variants.** Only variants whose provider is registered and that support the requested model are candidates. If `control` serves `gpt-4o` and `challenger` doesn't, every `gpt-4o` request goes to `control` regardless of the configured weight — the effective split differs per model when variants serve different model sets. +- **The variant label is logged per request** for analytics — this is how you attribute observed outcomes back to `control` vs. `challenger` after the fact. +- **`weight: 0` drains a variant.** A drained variant is excluded from the draw entirely; it can never win, even if it's technically eligible. +- **`ab_variants[].weight` is what counts — `targets[].weight` is ignored under this mode.** `targets[].weight` only matters under `loadbalance`; don't expect it to influence the A/B split. +- **Weights are relative, not percentages.** `weight: 70` and `weight: 30` behave identically to `weight: 7` and `weight: 3` — only the ratio matters. +- **This is a pool mode, but the draw itself is not retried.** Once a variant is drawn, that variant's target leads; on failure the pipeline advances through the remaining configured targets like any pool mode. There is no re-draw — a failed request does not get a second chance at a different variant within the same routing decision. +- **A negative weight or an all-zero weight set is a load error**, not a runtime failure — `ferrogw validate` and gateway startup reject it before traffic is served. + +## Related + +- [Routing overview](/routing) +- [Load balance](/routing/loadbalance) +- [Fallback](/routing/fallback) +- [Cost-optimized](/routing/cost-optimized) +- [Configuration reference](/getting-started/configuration) diff --git a/docs/routing/conditional.mdx b/docs/routing/conditional.mdx new file mode 100644 index 0000000..7f3a370 --- /dev/null +++ b/docs/routing/conditional.mdx @@ -0,0 +1,62 @@ +--- +title: Conditional routing strategy +description: "Configure strategy.mode: conditional in Ferro Labs AI Gateway to pin models to targets by exact-name or prefix match, first match wins, targets[0] as fallback." +keywords: [conditional routing, model routing, strategy.mode conditional, model_prefix, ai-gateway routing, deterministic model routing, named mode routing] +--- + +Conditional routes each request by matching its declared `model` field against rules you write, not by weight, latency, or cost. It optimizes for **deterministic model-to-provider pinning** — "this model always goes to this backend, no exceptions" — the shape a compliance or contractual requirement needs, where a pool mode's willingness to substitute a different provider is the wrong behavior, not a convenience. Set `strategy.mode: conditional` to use it. + +## Behaviour + +`conditional` is a **named mode**: the pipeline commits to the target the first matching rule names and reports that target's failure, rather than trying the remaining targets (contrast with `fallback`, `loadbalance`, `least-latency`, `cost-optimized`, and `ab-test`, which are pool modes that advance past a failure). `Conditional.SelectTargets` (`internal/strategies/conditional.go`) evaluates `strategy.conditions[]` in declared order; the first rule whose `key`/`value` matches the request wins, and when none matches, the configured fallback target is used instead. The matched (or fallback) target leads the returned list, followed by every configured target de-duplicated — that tail is substitution-before-commit only (an open circuit or a surface-incapable target is swapped for a healthy one before the pipeline commits), never a failover path. + +The matcher key is a **closed set**: `model` (`req.Model == value`, exact) and `model_prefix` (`strings.HasPrefix(req.Model, value)`). There is no dynamic arm in the switch, so a `key` outside those two is rejected by `config.ValidateConfig` at load — a typo fails startup, not a live request. In the gateway's own wiring the no-match fallback is `targets[0]`, but that's a property of how `buildStrategy` constructs the strategy, not something `Conditional` itself enforces — it always asks `matchTarget`, never assumes the head of a list. + +## Config keys + +| Key | Type | Default | Description | +|---|---|---|---| +| `strategy.mode` | string | — | Set to `conditional`. | +| `strategy.conditions[].key` | string | — | `model` (exact match) \| `model_prefix` (prefix match via `strings.HasPrefix`). Closed set — an unrecognized value is a `ferrogw validate` / startup error, not a request-time failure. | +| `strategy.conditions[].value` | string | — | The exact model name (`key: model`) or prefix (`key: model_prefix`) to match against the request's `model` field. | +| `strategy.conditions[].target_key` | string | — | Must name a configured `targets[].virtual_key`; validated at load. | + +## Minimal working YAML + +```yaml +strategy: + mode: conditional + conditions: + - key: model + value: gpt-4o + target_key: openai + - key: model_prefix + value: claude- + target_key: anthropic + +targets: + - virtual_key: openai # targets[0] doubles as the no-match fallback + - virtual_key: anthropic +``` + +## When to use + +- Deterministic model-to-provider pinning: a specific model must always be served by a specific backend, for compliance, contractual, or data-residency reasons — not "prefer this provider," but "only this provider." +- Routing whole model families by prefix (`claude-`, `gpt-`, `gemini-`) to their native provider without listing every model name individually. +- Any setup where `fallback`'s or `loadbalance`'s willingness to substitute a different provider on failure is the behavior you need to prevent, not add. + +## Gotchas + +- **A model the matched target doesn't serve is `404 model_not_found`, even when another configured target serves it.** This is a named mode: the rule is a decision about which target handles this model, not a preference among several. Reaching the other target means writing another rule that names it — the tail of `targets[]` is never tried as a fallback for this. +- **`GET /v1/models` mirrors the same restriction.** A model missing from that listing under this mode is the signal that a rule is needed for it, before a request ever hits the 404. +- **The no-match fallback is `targets[0]`.** `NewConditional`'s constructor argument is a `fallback` value passed explicitly by `buildStrategy`, and in the gateway's wiring that argument is `targets[0]` — so put the target you want unmatched models to land on first. +- **The condition-key set is closed on purpose.** `model` and `model_prefix` are the only two matcher arms; anything else — a typo, a field name from another mode — is rejected at load by `config.ValidateConfig`, not silently ignored or routed to the fallback. +- **Circuit state is the one thing that still moves the choice.** If the matched (or fallback) target's circuit is open, the pipeline substitutes a healthy target from the tail before committing, and traffic returns to the matched target once its circuit closes. This is unrelated to, and doesn't reintroduce, failover on an ordinary request failure. + +## Related + +- [Routing overview](/routing) +- [Content-based](/routing/content-based) +- [Fallback](/routing/fallback) +- [Single](/routing/single) +- [Configuration reference](/getting-started/configuration) diff --git a/docs/routing/content-based.mdx b/docs/routing/content-based.mdx new file mode 100644 index 0000000..82ed55c --- /dev/null +++ b/docs/routing/content-based.mdx @@ -0,0 +1,64 @@ +--- +title: Content-based routing strategy +description: "Configure strategy.mode: content-based in Ferro Labs AI Gateway to route on user-prompt text via prompt_contains, prompt_not_contains, and Go-regex rules." +keywords: [content-based routing, prompt-based routing, prompt_contains, prompt_regex, prompt_not_contains, ai-gateway strategy.mode, LLM prompt routing] +--- + +Content-based routes each request by inspecting the text of its **user-role** prompt messages, not its declared `model` field. It optimizes for **prompt-aware model selection** — code-shaped prompts to a coding model, prompts mentioning "translate" to a translation-tuned model, everything else to a general default — a decision `conditional` routing can't make because it only ever sees the model name. Set `strategy.mode: content-based` to use it. + +## Behaviour + +`content-based` is a **named mode**: the pipeline commits to the target the first matching rule names and reports that target's failure, rather than trying the remaining targets (contrast with `fallback`, `loadbalance`, `least-latency`, `cost-optimized`, and `ab-test`, which are pool modes that advance past a failure). `ContentBased.SelectTargets` (`internal/strategies/contentbased.go`) evaluates `strategy.content_conditions[]` in declared order over **user-role messages only** — system and assistant content is never inspected — and the first rule that matches wins. The matched rule's target leads the returned list, followed by every configured target de-duplicated; the rest of the list is substitution-before-commit only (an open circuit or a surface-incapable target is swapped for a healthy one before the pipeline commits), never a failover path. With no rule matching, `SelectTargets` returns `targets[]` in declared order, so `targets[0]` is the no-match fallback. + +Because the match happens against request content, `GET /v1/models` under this mode is **representative, not exact**: it advertises what a request carrying no content would get — the no-match fallback target's models — so a model only a matched rule reaches is not listed there. + +## Config keys + +| Key | Type | Default | Description | +|---|---|---|---| +| `strategy.mode` | string | — | Set to `content-based`. | +| `strategy.content_conditions[].type` | string | — | `prompt_contains` \| `prompt_not_contains` \| `prompt_regex`. Closed set — an unrecognized value is a `ferrogw validate` / startup error, not a request-time failure. | +| `strategy.content_conditions[].value` | string | — | The substring (matched case-insensitively) for `prompt_contains`/`prompt_not_contains`, or the Go (RE2) regular expression for `prompt_regex`. | +| `strategy.content_conditions[].target_key` | string | — | Must name a configured `targets[].virtual_key`; validated at load. | + +## Minimal working YAML + +```yaml +strategy: + mode: content-based + content_conditions: + - type: prompt_regex + value: "(?i)\\b(code|function|class|implement)\\b" + target_key: deepseek + - type: prompt_contains + value: translate + target_key: gemini + +targets: + - virtual_key: openai # no-match fallback (targets[0]) + - virtual_key: deepseek + - virtual_key: gemini +``` + +## When to use + +- Prompt-aware model selection: route coding questions to a code-specialized model, translation requests to a cheaper or translation-tuned model, and everything else to a general default — decided by what the user actually asked, not by which model name the client requested. +- Keeping content that mentions a sensitive topic on a specific, contractually-approved backend, using `prompt_contains` (or excluding everything that mentions it from a cheaper backend with `prompt_not_contains`). +- Any routing decision `conditional` can't express, because `conditional` matches only `model`/`model_prefix` and never looks at message content. + +## Gotchas + +- **Only user-role messages are inspected.** System prompts and prior assistant turns never match, by any of the three rule types. +- **`prompt_not_contains` matches broadly, so rule order matters a lot.** It's true for *any* prompt that lacks the value — an early, broad `prompt_not_contains` rule can shadow every more-specific rule declared after it. Put narrow rules first. +- **A model the matched target doesn't serve is `404 model_not_found`, even when another configured target serves it.** This is a named mode: the rule is a decision about which target handles this content, not a preference among several. If a matched target needs to serve more models, add them there (or `targets[].models`) rather than expecting the tail of the list to cover it. +- **`GET /v1/models` is representative, not exact, under this mode.** It shows only what the no-match fallback (`targets[0]`) serves, since there's no request content to evaluate rules against at listing time — a model reachable only through a matched rule won't appear. +- **Regex is Go RE2 syntax.** No backreferences, no lookahead/lookbehind. An invalid pattern is caught at config load (`config.ValidateConfig` compiles every `prompt_regex` pattern the same way `NewContentBased` does), so a bad pattern fails startup rather than misrouting silently. +- **Circuit state is the one thing that still moves the choice.** If the matched target's circuit is open, the pipeline substitutes a healthy target from the tail before committing — traffic returns to the matched target once its circuit closes. This is unrelated to, and doesn't reintroduce, failover on ordinary request failure. + +## Related + +- [Routing overview](/routing) +- [Conditional](/routing/conditional) +- [Fallback](/routing/fallback) +- [Cost-optimized](/routing/cost-optimized) +- [Configuration reference](/getting-started/configuration) diff --git a/docs/routing/cost-optimized.mdx b/docs/routing/cost-optimized.mdx new file mode 100644 index 0000000..f0f0a04 --- /dev/null +++ b/docs/routing/cost-optimized.mdx @@ -0,0 +1,68 @@ +--- +title: Cost-optimized routing strategy +description: "Configure strategy.mode: cost-optimized in Ferro Labs AI Gateway to route to the cheapest priced target by estimated input cost, using unpriced_strategy." +keywords: [cost-optimized routing, AI gateway cost routing, LLM cost optimization, unpriced_strategy, cheapest model routing, model catalog pricing, ai-gateway strategy.mode] +--- + +Cost-optimized routes each request to the cheapest model-compatible target, ranked by **estimated input cost** from the built-in model catalog. It optimizes for **spend** — not failover order (`fallback`), distribution (`loadbalance`), or latency (`least-latency`). Set `strategy.mode: cost-optimized` to use it. + +## Behaviour + +`cost-optimized` is a **pool mode**: on a request failure, the pipeline advances to the next candidate in the ranked order rather than reporting the failure back to the caller. `targets[].retry` still governs how many times any one target is retried before the pipeline moves on. + +On each request, `CostOptimized.SelectTargets` (`internal/strategies/costoptimized.go`) filters `targets[]` down to those whose provider is registered and whose `SupportsModel(req.Model)` returns true, then estimates the prompt token count at roughly **4 characters per token** (a routing heuristic, not a billing figure) and prices `/` for each compatible target through the model catalog. Candidates are ranked by ascending estimated input cost — the cheapest target leads. `strategy.unpriced_strategy` decides which cataloged-but-unpriced candidates are allowed to rank at all (see below). Candidates the catalog doesn't recognize as that model (`ModelFound: false`) never rank, regardless of `unpriced_strategy`. + +## Config keys + +| Key | Type | Default | Description | +|---|---|---|---| +| `strategy.mode` | string | — | Set to `cost-optimized`. | +| `strategy.unpriced_strategy` | string | `fallback` | `fallback` \| `skip` \| `allow`. Governs how compatible targets the catalog knows about but has no price for are treated during ranking. | + +### `unpriced_strategy` in detail + +| Value | Ranking | When nothing ranks | +|---|---|---| +| `fallback` (default) | Only **priced** candidates rank, cheapest first. | The compatible candidates lead the returned order in **declared order** (not `targets[0]`, the first compatible one) — no error. | +| `skip` | Only **priced** candidates rank, cheapest first. | Returns an **error** wrapping `core.ErrNoCapableProvider` — the only strategy mode that can return a non-nil error from `SelectTargets`. | +| `allow` | Every model-compatible candidate ranks, priced or not. A missing price sorts as **`$0`** — the cheapest possible value — so an unpriced target wins the draw over any priced one. | — (there is always at least one ranked candidate if any target is compatible). | + +`ferrogw validate` accepts only an empty value (the default), `fallback`, `skip`, or `allow` for `unpriced_strategy` — anything else fails at load time, so an invalid value never reaches a live request. + +## Minimal working YAML + +```yaml +strategy: + mode: cost-optimized + unpriced_strategy: fallback + +targets: + - virtual_key: deepseek + - virtual_key: openai + - virtual_key: anthropic +``` + +## When to use + +- Multiple providers serve the same (or equivalent) model and the catalog carries pricing for them — let cost decide instead of hand-picking a provider. +- You want spend minimized automatically as new, cheaper models or providers are added to `targets[]`, without a config change. +- Combined with `retry` on each target, so a failed cheapest-provider attempt falls through to the next-cheapest rather than erroring. + +## Gotchas + +- **Input cost only, and estimated.** Ranking uses only prompt/input tokens, priced from the catalog's per-1M-input-token field, estimated at ~4 characters per token. Output pricing is never considered, so a target that's cheap on input but expensive on output can still be picked. Do not read the ranking as a billing-accurate number — it's a routing heuristic. +- **`allow` makes unpriced look free.** Under `unpriced_strategy: allow`, any compatible target the catalog has no price for is treated as `$0` and wins the draw over every priced target, every time. This is useful for a self-hosted/local target you always want preferred, but a surprise for one you didn't intend to be prioritized. +- **`skip` narrows what `/v1/models` advertises.** Under `unpriced_strategy: skip`, targets with no catalog price are excluded from ranking entirely; if a request's model has no priced target, the request errors rather than falling through. `targets[].models` entries are unpriced by construction (they're operator-declared, not catalog data), so they're excluded here too. +- **Default (`fallback`) only reaches unpriced targets as a last resort.** A self-hosted or otherwise-unpriced target under the default strategy is used only when **no** compatible target in `targets[]` has any catalog price at all — not merely when it's cheaper. +- **This is a pool mode.** The pipeline advances past a failed or open-circuit target to the next one in the ranked order — `cost-optimized` gets failover "for free" as a side effect of the mode family, not because of anything cost-specific. +- **Only model-compatible targets are eligible.** A target whose provider isn't registered, or that doesn't serve the requested model, is filtered out before pricing is even looked up. +- **Ranking is per-request.** Because it depends on the requested model and that request's estimated prompt size, the effective order can differ from one request to the next through the same config — it is not a static, once-computed target list. + +## Related + +- [Routing overview](/routing) +- [Fallback](/routing/fallback) +- [Load balance](/routing/loadbalance) +- [Least-latency](/routing/least-latency) +- [A/B test](/routing/ab-test) +- [Configuration reference](/getting-started/configuration) diff --git a/docs/routing/fallback.mdx b/docs/routing/fallback.mdx new file mode 100644 index 0000000..7bb86d6 --- /dev/null +++ b/docs/routing/fallback.mdx @@ -0,0 +1,60 @@ +--- +title: Fallback routing strategy +description: "Configure strategy.mode: fallback in Ferro Labs AI Gateway to try targets in priority order with per-target retry, full-jitter backoff, and Retry-After support." +keywords: [fallback routing, failover, AI gateway fallback, ai-gateway strategy.mode, retry backoff, priority order routing] +--- + +Fallback tries targets in the order you declare them, advancing to the next target when the current one fails. It optimizes for **availability**: a failed request is answered by the next-best provider instead of being reported as a failure — not for spreading load (`loadbalance`), speed (`least-latency`), or cost (`cost-optimized`). Set `strategy.mode: fallback` to use it. + +## Behaviour + +`fallback` is a **pool mode** — the only mode whose entire purpose is advancing past a failed target in the order you declared. `Fallback.SelectTargets` (`internal/strategies/fallback.go`) returns every `targets[].virtual_key` in declared order, built once at construction; there is no per-request selection logic, no model filtering, and `targets[].weight` is ignored under this mode. The gateway pipeline (`routeTargets`, `gateway_pipeline.go`) walks that order: when the head target fails in a way its retry policy won't absorb, the walk advances to the next declared target rather than reporting the failure back to the caller. + +`targets[].retry` governs how many times **one** target is re-asked before the pipeline moves on — it applies under every routing mode, not only `fallback`. Fallback holds no retry policy of its own; advancing to the *next* target is fallback's behaviour alone. An open-circuit or surface-incapable target is skipped over before the pipeline commits to it, under every mode. `SelectTargets` never returns `nil` (declaring at least one target is required to build the strategy), but if every candidate's circuit is open the pipeline still attempts one and answers `503` rather than `404`. + +## Config keys + +| Key | Type | Default | Description | +|---|---|---|---| +| `strategy.mode` | string | — | Set to `fallback`. | +| `targets[].retry.attempts` | int | `1` (no retry) — omitted or `<= 0` normalizes to `1` | Maximum attempts against this one target before the pipeline advances to the next. Read per target, honored under every strategy mode. | +| `targets[].retry.on_status_codes` | []int | transport errors + `408`, `429`, and `5xx` | Restricts retries to these HTTP status codes. Any other 4xx is treated as a deterministic client error and is never retried. | +| `targets[].retry.initial_backoff_ms` | int | `100` | Base for exponential backoff with full jitter: the wait before attempt *N* is drawn uniformly from `[0, initial_backoff_ms * 2^(N-1))`. An upstream `Retry-After` header, when present, wins over the computed wait. | + +## Minimal working YAML + +```yaml +strategy: + mode: fallback + +targets: + - virtual_key: openai + retry: + attempts: 3 + on_status_codes: [429, 502, 503] + initial_backoff_ms: 100 + - virtual_key: anthropic +``` + +## When to use + +- A primary provider plus one or more backups, where getting an answer from *someone* matters more than getting it from the preferred provider every time. +- Declared priority matters: you want traffic to favor a specific provider (cheapest, fastest, contractually preferred) and spill to the next one only when the preferred provider fails. +- You want failover without hand-tuning weights (`loadbalance`) or relying on measured latency (`least-latency`). + +## Gotchas + +- **Fallback holds no retry policy of its own.** `targets[].retry.*` is the pipeline's config, resolved per target and applied identically under every routing mode. What's specific to `fallback` is only the decision to advance to the *next* target once that target's retry budget is spent. +- **Declared order is priority order — there are no weights.** `targets[].weight` is read only by `loadbalance`; under `fallback`, the first entry in `targets[]` is always tried first. +- **Retries never burn budget against cancellation, deadline, an open circuit, or saturation.** `context.Canceled`, `context.DeadlineExceeded`, an open-circuit error, and `ErrProviderSaturated` (a full concurrency queue) are never retried — the pipeline treats them as an immediate signal to move to the next target rather than as something a retry could fix. +- **A `Retry-After` hint longer than 30 seconds abandons the target instead of waiting on it.** The gateway won't hold a request open past that cap; it advances to the next target immediately and logs that the target was abandoned. +- **Only deterministic-retry statuses are retried by default.** With no `on_status_codes` set, retries apply to transport failures plus `408`, `429`, and `5xx`. Every other 4xx (e.g. `400`, `401`, `404`) is treated as a client error that a retry against the same target cannot fix. +- **An open-circuit target is skipped before commitment, same as every other mode.** If every declared target's circuit is open, the pipeline still attempts one rather than reporting a routing-level 404 — the caller sees `503`. + +## Related + +- [Routing overview](/routing) +- [Load balance](/routing/loadbalance) +- [Least-latency](/routing/least-latency) +- [Cost-optimized](/routing/cost-optimized) +- [Configuration reference](/getting-started/configuration) diff --git a/docs/routing/least-latency.mdx b/docs/routing/least-latency.mdx new file mode 100644 index 0000000..8c37fd2 --- /dev/null +++ b/docs/routing/least-latency.mdx @@ -0,0 +1,57 @@ +--- +title: Least-latency routing strategy +description: "Configure strategy.mode: least-latency to route to the target with the lowest observed p50 latency, with cold-start shuffling for unseen targets." +keywords: [least-latency routing, p50 latency routing, AI gateway latency-based routing, ai-gateway strategy.mode, cold-start profiling, latency tracker] +--- + +Least-latency routes to the compatible target with the lowest observed **p50** latency. It optimizes for **total response time** across interchangeable providers — not for cost (`cost-optimized`), declared failover order (`fallback`), or an even traffic split (`loadbalance`). Set `strategy.mode: least-latency` to use it. + +## Behaviour + +`least-latency` is a **pool mode**: on a request failure, the pipeline advances to the next candidate in the ordered list rather than reporting the failure back to the caller. `targets[].retry` still governs how many times any one target is retried before the pipeline moves on. + +On each request, `LeastLatency.SelectTargets` (`internal/strategies/leastlatency.go`) filters `targets[]` down to those whose provider is registered and whose `SupportsModel(req.Model)` returns true, then splits them by whether the in-process `latency.Tracker` holds any samples for that target. Targets with **no recorded samples are shuffled to the front** — cold-start profiling, so a newly added target gets tried rather than starved by an established leader — followed by sampled targets sorted **ascending by p50**, followed by any remaining declared targets. The pipeline commits to the head of that order first; the rest stand in as substitutes before committing (an open circuit, for example) and, because this is a pool mode, as the next attempt if the head fails. `SelectTargets` returns `nil` (the caller reports 404 `model_not_found`) when no configured target serves the requested model. + +The sample the tracker records is **total wall-clock for the request**, not time-to-first-token and not provider health. Generation length dominates it: a provider that answers the same prompt at greater length reads as "slower" than a terse one on identical hardware, and a model whose replies are simply longer loses to a model whose replies are shorter regardless of service speed. That is the intended trade — wall-clock is what a caller actually waits for, so ranking on it optimizes the number the caller experiences. Do not read the ordering as a health claim; `/health`, `/readyz`, and the circuit-breaker metric answer that question instead. + +## Config keys + +| Key | Type | Default | Description | +|---|---|---|---| +| `strategy.mode` | string | — | Set to `least-latency` (note the hyphen). | + +There are no other strategy-level keys for this mode — the latency tracker is internal process state, not configurable, and `weight` on `targets[]` is ignored. + +## Minimal working YAML + +```yaml +strategy: + mode: least-latency + +targets: + - virtual_key: groq + - virtual_key: openai + - virtual_key: anthropic +``` + +## When to use + +- Total response time is the objective, and the configured targets serve equivalent or interchangeable models across providers. +- You want the gateway to steer traffic toward whichever provider is currently answering fastest, without hand-tuning weights or a declared priority order. + +## Gotchas + +- **Measures wall-clock, not health or TTFT.** The ranking reflects how long a full response took, generation length included — it is not a claim about provider uptime or how fast the first token arrived. Use `/health`, `/readyz`, and the circuit-breaker metric to answer health questions. +- **Unseen targets jump the queue by design.** A target with no recorded samples is shuffled to the front ahead of every sampled target, so adding a new target to the pool means it briefly absorbs traffic while it's profiled — this is intentional cold-start behaviour, not a bug. +- **Samples reset with the process.** The tracker is in-memory only; a restart, redeploy, or rolling update wipes all latency history, so every target starts "unseen" again and cold-start shuffling kicks back in. +- **`weight` is ignored.** `targets[].weight` has no effect under this mode — it is read only by `loadbalance`. +- **This is a pool mode.** The pipeline advances past a failed or open-circuit target to the next one in p50 order; failover falls out of the mode family, not out of anything latency-specific. +- **Only model-compatible targets are eligible.** A target whose provider isn't registered, or that doesn't serve the requested model, is filtered out before latency ordering is applied. + +## Related + +- [Routing overview](/routing) +- [Load balance](/routing/loadbalance) +- [Fallback](/routing/fallback) +- [Cost-optimized](/routing/cost-optimized) +- [Configuration reference](/getting-started/configuration) diff --git a/docs/routing/loadbalance.mdx b/docs/routing/loadbalance.mdx new file mode 100644 index 0000000..4e5a55d --- /dev/null +++ b/docs/routing/loadbalance.mdx @@ -0,0 +1,57 @@ +--- +title: Load balance routing strategy +description: "Configure strategy.mode: loadbalance in Ferro Labs AI Gateway for weighted-random traffic across model-compatible targets, plus draining a target via weight: 0." +keywords: [load balance routing, weighted random routing, AI gateway load balancing, weight 0 drain, ai-gateway strategy.mode, targets weight] +--- + +Load balance spreads traffic across two or more targets by weighted random selection. It optimizes for **distributing** load across interchangeable providers — not for failover order (`fallback`), latency (`least-latency`), or cost (`cost-optimized`). Set `strategy.mode: loadbalance` to use it. + +## Behaviour + +`loadbalance` is a **pool mode**: on a request failure, the pipeline advances to the next target in the rotated order rather than reporting the failure back to the caller. `targets[].retry` still governs how many times any one target is retried before the pipeline moves on. + +On each request, `LoadBalance.SelectTargets` (`internal/strategies/loadbalance.go`) filters `targets[]` down to those whose provider is registered and whose `SupportsModel(req.Model)` returns true, then picks a weight-biased starting index (`weightedStartIndex`, `internal/strategies/targetorder.go`) and returns the full compatible list **rotated** from that index. The first key in the returned order is the weighted pick the pipeline commits to first; the rest stay available as substitutes the pipeline may use before committing — for example when the picked target's circuit is open. `SelectTargets` returns `nil` (the caller reports 404 `model_not_found`) when no compatible target exists, or when every compatible target has been drained to weight `0`. + +## Config keys + +| Key | Type | Default | Description | +|---|---|---|---| +| `strategy.mode` | string | — | Set to `loadbalance`. | +| `targets[].weight` | float64 | `0` if omitted (YAML `omitempty`) | This target's relative share of traffic. Ignored by every other routing mode. `0` means the target receives zero traffic — it can never be the rotation's start index. | + +## Minimal working YAML + +```yaml +strategy: + mode: loadbalance + +targets: + - virtual_key: openai + weight: 0.7 + - virtual_key: anthropic + weight: 0.3 +``` + +## When to use + +- Spreading load across two or more providers serving the same (or overlapping) models, for cost or capacity reasons. +- Gradually shifting traffic during a migration by adjusting weights across deploys. +- Draining a target to zero traffic before revoking its credential or removing it from `targets[]`. + +## Gotchas + +- **Weights are relative, not percentages.** `weight: 0.7` and `weight: 0.3` behave identically to `weight: 7` and `weight: 3` — only the ratio between targets matters. +- **Omitting `weight` yields `0`, not an even split.** Because `weight` uses YAML `omitempty`, a target with no `weight` key defaults to `0.0` — silent zero traffic. This is an easy foot-gun: set `weight` explicitly on every target under `loadbalance`. +- **An all-zero or negative weight set is rejected at load.** `ferrogw validate` (and gateway startup) reject a negative weight outright and reject a config where every target's weight sums to zero — but one *forgotten* weight on an otherwise-valid config passes validation and silently drains just that target. +- **This is a pool mode.** The pipeline advances past a failed or open-circuit target to the next one in the rotated order; a `loadbalance` config gets failover "for free" as a side effect of the mode family, not because of anything weight-specific. +- **Only model-compatible targets are eligible.** A target whose provider isn't registered, or that doesn't serve the requested model, is filtered out before weighting is applied — it never wins the draw regardless of its configured weight. +- Selection uses `math/rand`, deliberately — this is a load-shaping decision, not a security-sensitive one. + +## Related + +- [Routing overview](/routing) +- [Fallback](/routing/fallback) +- [Least-latency](/routing/least-latency) +- [Cost-optimized](/routing/cost-optimized) +- [A/B test](/routing/ab-test) +- [Configuration reference](/getting-started/configuration) diff --git a/docs/routing/overview.mdx b/docs/routing/overview.mdx new file mode 100644 index 0000000..6a0800f --- /dev/null +++ b/docs/routing/overview.mdx @@ -0,0 +1,123 @@ +--- +title: Routing overview +description: "How the Ferro Labs AI Gateway routes requests: eight strategies that order targets, the strategy and targets config, retry, circuit breaking, and failover." +keywords: [AI gateway routing, LLM routing strategies, fallback routing, load balancing, cost-optimized routing, least-latency routing, conditional routing, circuit breaker] +slug: /routing +--- + +Routing is how the Ferro Labs AI Gateway decides which provider answers a request. One **strategy** governs the whole gateway: it takes each request and returns an ordered list of **targets** to try. A strategy decides *order* and nothing else — retry, circuit breaking, failure classification and capability checks all live once in the request pipeline, so chat, streaming, embeddings and image generation route identically. Ferro Labs ships eight strategies across two families. + +## What a strategy is + +A strategy implements a single method, `SelectTargets(req)`, which returns the ordered virtual keys it would try for a request, most-preferred first, followed by the remaining configured targets. That ordering is the whole of its job. + +This follows Envoy's split: the route table and load balancer say *where* a request may go, while the retry policy hangs off the route and the circuit breaker off the cluster. In the gateway, the strategy names order; the pipeline (`routeTargets`) owns retry, the circuit breaker, the concurrency limiter, failure classification and "nothing here can serve this". Stating order once and executing it once is why a config orders its targets the same way whether the request streams or not. + +- A **nil** result means no configured target serves the requested model — the caller reports `404 model_not_found`. +- Only `cost-optimized` (in `skip` mode) can return an error; every other strategy returns a nil error. + +## Configure routing + +Routing is two top-level blocks in `config.yaml` (or JSON): a `strategy:` block that sets the `mode` plus that mode's own keys, and a flat `targets:` list. There is **no per-route nesting** — one strategy governs the entire gateway. + +```yaml +strategy: + mode: fallback # single | fallback | loadbalance | least-latency | + # cost-optimized | conditional | content-based | ab-test + +targets: + - virtual_key: openai + retry: + attempts: 3 + - virtual_key: anthropic +``` + +Mode-specific keys live under `strategy:` alongside `mode`: `conditions[]` (conditional), `content_conditions[]` (content-based), `ab_variants[]` (ab-test) and `unpriced_strategy` (cost-optimized). Each is documented on that strategy's own page. + +:::info `targets` is an allowlist +Setting a provider's credentials registers it, but a provider only routes if it appears in `targets[]`. A request for a model no listed target serves is `404 model_not_found`, even when a registered-but-unlisted provider could have served it. To stop routing to a provider, remove its target. +::: + +### The `targets[]` entry + +Every entry names one provider registration and optionally attaches per-target resilience. `virtual_key` is the only required field. + +| Key | Type | Default | Description | +|---|---|---|---| +| `virtual_key` | string | — (required) | Names a provider registration, typically the provider id. `ferrogw validate` rejects a key no built-in provider matches. | +| `weight` | float64 | `0` | Relative share under `loadbalance` **only**; ignored by every other mode. `0` means zero traffic — the drain lever. | +| `models` | []string | — | Extra exact model IDs this target serves, added to the routing index and `/v1/models`. **Additive only** (never hides what a target already serves), **no wildcards** (rejected at load). | +| `retry` | object | single attempt | Per-target retry policy. Applies under **every** mode — see below. | +| `circuit_breaker` | object | none | Per-target breaker. One per `virtual_key`, shared by all four surfaces. | +| `concurrency` | object | unlimited | In-flight bound with a queue; overflow fails fast with `429`. | + +**`retry`** — how many times *one* target is re-asked (advancing to the *next* target is the routing mode's job, not retry's): + +| Key | Type | Default | Description | +|---|---|---|---| +| `attempts` | int | `1` | Max attempts against this target. `1` = no retry; the whole `retry` block is optional. | +| `on_status_codes` | []int | transport errors + `408`/`429`/`5xx` | Restrict retryable statuses. Other `4xx` are deterministic client errors and are never retried. | +| `initial_backoff_ms` | int | `100` | Base for exponential backoff with full jitter: wait is drawn from `[0, initial * 2^(attempt-1))`. An upstream `Retry-After` wins; a hint over 30s abandons the target. | + +**`circuit_breaker`** — a target whose circuit is open is skipped during selection (see below): + +| Key | Type | Default | Description | +|---|---|---|---| +| `failure_threshold` | int | `5` | Consecutive failures before the circuit opens. | +| `success_threshold` | int | `1` | Consecutive half-open successes required to close. | +| `max_half_threshold` | int | `1` | Concurrent probes allowed while half-open. | +| `timeout` | duration | `30s` | How long the circuit stays open before going half-open (e.g. `"30s"`). | + +**`concurrency`** — bounds simultaneous in-flight requests (a streaming request holds its slot until the stream ends): + +| Key | Type | Default | Description | +|---|---|---|---| +| `max_concurrency` | int | unlimited | Max in-flight requests to this target; capped at `10000`. | +| `queue_size` | int | default queue | Requests allowed to wait for a slot; beyond it the request is shed with `429`. | + +## Two mode families + +The pipeline splits the modes by what their leading candidate *means*, and that decides whether moving off it is a repair or a betrayal. + +| Family | Modes | On a target failure | +|---|---|---| +| **Pool** | `fallback`, `loadbalance`, `least-latency`, `cost-optimized`, `ab-test` | Advances to the next candidate. | +| **Named** | `single`, `conditional`, `content-based` | Commits to the chosen target and reports its failure. | + +A **pool** mode picks its head for a reason that is about the pool rather than the individual target — spread the load, take the cheapest, take the fastest, split the traffic — from targets the operator declared interchangeable, so carrying a failed request to a sibling is what was asked for. A **named** mode picks its head because something named that target specifically (`single` names it; a `conditional` or `content-based` rule matched it), so serving from anyone else would demote the rule to a suggestion. + +## What every mode shares + +Regardless of family: + +- **Retry re-asks the same target.** `targets[].retry` is honoured under every mode. It never advances to a different target — only pool modes do that. +- **An open circuit is skipped.** Before committing, a target whose breaker is open (or that cannot serve the request's surface at all) is passed over in favour of the next candidate — even under the named modes, where the operator named one target on purpose. +- **All circuits open → `503`.** When every candidate's circuit is open the request is still attempted, the breaker refuses it, and the caller gets `503 upstream_unavailable`. That is deliberately different from `404`: "everything that serves this model is down" is not the same answer as "nothing serves this model". +- **No candidate serves the model → `404`.** When no configured target serves the requested model, nothing is attempted and the caller gets `404 model_not_found`. + +## v1.4 breaking changes + +Two routing behaviours changed and may need config edits when upgrading: + +- **`targets[].retry` now applies under every mode.** It used to run only under `fallback`. If a target carried a `retry` block for use under `fallback` and you now run another mode, set `attempts: 1` to keep the old single-attempt behaviour. +- **`targets` is an allowlist.** A model is routable only through a listed target (plus that target's `models`, the catalog and live discovery). A registered provider that no target names does not route; an unowned model is `404 model_not_found`. + +## The eight strategies + +| Strategy | Family | Use when | +|---|---|---| +| [Single](/routing/single) | Named | You have one provider and want the gateway as a pure governance and observability layer. | +| [Fallback](/routing/fallback) | Pool | A primary plus backups, where a failed request should be answered by someone else rather than reported. | +| [Load balance](/routing/loadbalance) | Pool | Spreading load across interchangeable providers, weight-shifted migration, or draining one (`weight: 0`) before removal. | +| [Least-latency](/routing/least-latency) | Pool | Total response time is the objective across equivalent models on several providers. | +| [Cost-optimized](/routing/cost-optimized) | Pool | Cost is the objective and the model catalog carries the prices. | +| [Conditional](/routing/conditional) | Named | Deterministic model→provider pinning, e.g. for compliance or contract reasons. | +| [Content-based](/routing/content-based) | Named | Prompt-aware selection: code to a code model, translation to a cheap one, sensitive content to a specific backend. | +| [A/B test](/routing/ab-test) | Pool | Comparing model quality or cost on a controlled live traffic split. | + +## Related + +- [Configuration](/getting-started/configuration) — full YAML/JSON reference +- [Providers](/providers) — what each provider serves and how to register it +- [Circuit breakers and failover](/operations/monitoring) — reading circuit state and metrics +- [Plugins](/plugins) — combine routing with guardrails, budgets and logging diff --git a/docs/routing/single.mdx b/docs/routing/single.mdx new file mode 100644 index 0000000..93dbe5e --- /dev/null +++ b/docs/routing/single.mdx @@ -0,0 +1,59 @@ +--- +title: Single routing strategy +description: "Configure strategy.mode: single, the default AI Gateway routing strategy — every request goes to targets[0], with no failover to any other configured target." +keywords: [single routing strategy, ai-gateway strategy mode, default routing mode, single provider gateway, no failover routing] +--- + +Single routes every request to one target — `targets[0]` — and nothing else. It optimizes for **simplicity and explicit control**: no weight math, no latency tracking, no per-request selection logic, just a provider behind a governance and observability layer. It's also the strategy you get by not choosing one: `strategy.mode: single`, or an empty/omitted `mode`, both build this strategy. + +## Behaviour + +`single` is a **named mode** — the opposite of a pool mode like `fallback` or `loadbalance`. `Single.SelectTargets` (`internal/strategies/single.go`) returns a one-element slice built once at construction, `[]string{targets[0].VirtualKey}`. It never looks at the request, never checks which model was asked for, and never sees `targets[1:]` at all — `buildStrategy` (`gateway_strategy.go`) constructs `Single` from `targets[0]` alone, so any additional entries in `targets[]` are configured but structurally invisible to this strategy. There is no failover: a failure at `targets[0]` — after `targets[].retry` exhausts its attempts against that same target — is the answer the caller gets. + +An open circuit breaker on `targets[0]` behaves differently here than under a pool mode, and it's worth being precise about it. The pipeline's open-circuit filter (`healthyKeys`, `gateway_pipeline.go`) only filters when the strategy handed it **more than one** candidate to choose from — with a single-element list, it returns that element unfiltered, open or not. So under `single`, an open circuit does not divert traffic anywhere: the request is still attempted, the circuit breaker itself refuses the call, and the gateway answers `503 upstream_unavailable`. That's a different outcome from `fallback`/`conditional`/`content-based`, where an open circuit on the lead target *is* passed over in favor of a healthy one in the same `targets[]` list — `single` has no such list to fall back into. + +## Config keys + +| Key | Type | Default | Description | +|---|---|---|---| +| `strategy.mode` | string | `single` | Selects this strategy. Leaving `mode` empty resolves to `single` too (`Config.Normalize`) — it's the gateway's zero-value default. | +| `targets[].virtual_key` | string | — (required) | Provider registration name. Only `targets[0].virtual_key` is ever read under this mode. | +| `targets[0].retry.attempts` | int | `1` (no retry) — omitted or `<= 0` normalizes to `1` | Attempts against `targets[0]` before giving up. Resolved by the pipeline, not the strategy, so it applies here exactly as it does under every other mode. | +| `targets[0].retry.on_status_codes` | []int | transport errors + `408`, `429`, `5xx` | Restricts retries to these HTTP status codes; other 4xx codes are deterministic client errors and are never retried. | +| `targets[0].retry.initial_backoff_ms` | int | `100` | Base for full-jitter exponential backoff between attempts against `targets[0]`. | +| `targets[0].circuit_breaker.*` | object | no breaker unless set (`failure_threshold: 5`, `success_threshold: 1`, `timeout: "30s"` if configured) | See the behaviour note above — with one candidate, an open circuit fails the request rather than routing around it. | +| `targets[].weight` | float64 | `0` | Ignored under `single`. Only `loadbalance` reads it. | + +## Minimal working YAML + +```yaml +strategy: + mode: single + +targets: + - virtual_key: openai + retry: + attempts: 3 +``` + +## When to use + +- One provider, no routing decisions to make — the gateway is purely a governance/observability layer (auth, plugins, metrics, request logging) in front of a single upstream. +- You want the simplest possible config while you evaluate the gateway, before adding a second provider. +- You're intentionally pinning all traffic to one provider for compliance or contractual reasons, and any deviation should be a config change you make on purpose — not something the gateway decides for you. + +## Gotchas + +- **Only `targets[0]` counts.** A second, third, or further entry in `targets[]` is accepted by `ferrogw validate` but is never selected, never health-checked by the strategy, and never receives traffic. If you meant to fail over to it, use `mode: fallback` instead. +- **Readiness doesn't know that either.** `/readyz` and startup logging check whether each *configured* target's provider is registered, not which one `single` would actually pick — so a config with a broken `targets[0]` and a perfectly healthy `targets[1]` still reports `200 ready`. Put the target you mean to use first. +- **No failover, by design.** A failure at `targets[0]` — once `retry.attempts` is exhausted — is the response the caller gets. Add `retry` to absorb transient errors against that one target; switch strategies if a hard outage should move traffic elsewhere. +- **An open circuit doesn't skip the target — it fails the request.** Because `SelectTargets` always returns exactly one key, the pipeline's open-circuit filter has nothing to filter between and never fires. The call is attempted anyway, the breaker refuses it, and the caller gets `503 upstream_unavailable` — even if another configured target could serve the model. This is easy to assume works like `fallback`'s circuit handling; it doesn't. +- **`weight` is silently accepted and silently ignored.** Setting `targets[0].weight` (or any other target's) does nothing under `single` — it only affects `loadbalance`. + +## Related + +- [Routing overview](/routing) +- [Fallback](/routing/fallback) +- [Load balance](/routing/loadbalance) +- [Configuration reference](/getting-started/configuration) +- [Monitoring](/operations/monitoring) diff --git a/docs/security/data-handling.mdx b/docs/security/data-handling.mdx index 516fc46..883aeb7 100644 --- a/docs/security/data-handling.mdx +++ b/docs/security/data-handling.mdx @@ -1,26 +1,115 @@ --- title: Data handling -description: Security and data handling in the Ferro Labs AI Gateway — storage backends, secret management, least privilege access, TLS termination, and provider credential isolation. -keywords: [AI gateway security, LLM data handling, PII protection AI, secure LLM proxy, credential isolation, AI gateway TLS] +description: "How the AI Gateway protects data: credential redaction by value, config secret withholding, MCP subprocess isolation, audit logging, and proxy path safety." +keywords: [AI gateway security, LLM data handling, credential redaction, PII protection AI, secure LLM proxy, credential isolation, AI gateway audit log] --- +The gateway holds provider credentials, admin keys, and MCP tokens in one process. This page covers how it keeps them out of logs, traces, config reads, and cached responses, and where the guarantees stop. + ## Storage backends -The gateway supports memory, SQLite, and Postgres stores for config, API keys, and request logs. +Config, admin API keys, and request logs each have their own configurable store. Sessions and the audit trail follow the API key store's backend. ```bash -export CONFIG_STORE_BACKEND=sqlite +export CONFIG_STORE_BACKEND=sqlite # memory (default) | sqlite | postgres export CONFIG_STORE_DSN=./ferrogw-config.db -export API_KEY_STORE_BACKEND=sqlite +export API_KEY_STORE_BACKEND=sqlite # memory (default) | sqlite | postgres export API_KEY_STORE_DSN=./ferrogw-keys.db -export REQUEST_LOG_STORE_BACKEND=sqlite +export REQUEST_LOG_STORE_BACKEND=sqlite # sqlite | postgres (unset = no persistence) export REQUEST_LOG_STORE_DSN=./ferrogw-requests.db ``` +Admin API keys are stored hashed (SHA-256), never in plaintext. The in-memory default is the documented default, but it means operator keys, dashboard sessions, and the audit trail are lost on restart — [`MASTER_KEY`](/guides/auth) is the only way back in when that happens. + +## Credential redaction by value + +Every credential the gateway holds — provider API keys read from the environment, admin keys, MCP headers and env values — is indexed once and matched **by exact value**, not by shape. Any string the gateway emits (log lines, OTel span attributes, sanitized proxy responses) has each occurrence replaced with a token naming its source, e.g. an upstream `401` body that echoes back the key the gateway presented becomes: + +``` +invalid api_key: [REDACTED:MISTRAL_API_KEY] +``` + +Only values of at least 12 characters are eligible — long enough to sit above every real provider key (the shortest is an AWS access key ID at 20 characters) and below no plausible placeholder, so ordinary log prose isn't accidentally redacted. Values containing whitespace, unresolved `${VAR}` references, or the literals `true`/`false` are never enrolled. + +Value-based redaction is exact but only covers credentials the gateway itself holds. As a backstop, a fixed set of shape-based patterns also runs — email addresses, JWTs, AWS access keys, bearer tokens, and several providers' key formats — to catch a credential the gateway never configured, such as a co-tenant's key echoed by a federated upstream error. This backstop is **best-effort**: it recognizes known, prefix-shaped formats and does not guarantee every credential-like string in arbitrary upstream text is caught. Treat redaction as defense in depth, not a substitute for scoping who can read logs, traces, and admin endpoints. + +:::note Where this applies +Route/embedding failure logs, OTel span error text under the default `observability.tracing.privacy_level: metadata`, and every non-2xx response the `/v1/*` pass-through proxy relays to the client are all filtered through this path before they leave the process. Setting `privacy_level: full` deliberately serves raw, unredacted error text on spans instead — reserve it for deployments where the trace backend is as trusted as the gateway's own logs. +::: + +## Outbound redirects are surfaced, not followed + +The gateway's own HTTP clients — used for provider calls, the `/v1/*` pass-through proxy, and AWS Bedrock credential lookups — do not follow `3xx` responses. A redirect is returned to the caller exactly as the upstream sent it, naming only the target's scheme and host (never its userinfo or query string, which could carry a credential). + +This exists because following a redirect would replay whatever credential the gateway injected for the original host against wherever the `Location` header points — including an attacker-controlled host on a misconfigured or compromised upstream. If a provider or proxy behind a `_BASE_URL` genuinely serves its API at a redirect, point the base URL at the destination directly rather than the redirecting one. The one exception is a Bedrock deployment that also supplies a custom TLS certificate bundle: the AWS SDK refuses a custom client in that case and keeps its own (redirect-following) HTTP client, and the gateway logs this at startup. + +MCP's Streamable HTTP transport applies the same policy: a `307` from an MCP server is refused with a hint naming the target rather than followed silently. + +## MCP subprocess environment isolation + +A `command`-based (stdio) MCP server is launched as a subprocess that does **not** inherit the gateway's environment. It receives only `PATH`, `HOME`, `LANG`, and `TMPDIR` (when set) plus whatever is listed explicitly under its `env:` block — so `OPENAI_API_KEY`, `MASTER_KEY`, and every other gateway credential are unreachable from an MCP subprocess unless deliberately passed through. + +```yaml +mcp_servers: + - name: filesystem + command: npx + args: ["-y", "@modelcontextprotocol/server-filesystem", "/data"] + env: + SOME_TOKEN: ${SOME_TOKEN} # only this token reaches the subprocess + required: false +``` + +`${VAR}` references in `mcp_servers[].headers` and `mcp_servers[].env` are resolved at client construction, not at config load — so the resolved secret is never written into the config-history store or served back by `GET /admin/config`. See [MCP](/guides/mcp) for the full config surface. + +## What `GET /admin/config` withholds + +A named field of the config schema (`virtual_key`, `mode`, `command`, `url`) is served intact with only its secret parts scrubbed to `[REDACTED]`. A **free-form map** — a surface the gateway hands to something else without knowing its shape — is withheld key by key instead: each entry comes back as `[REDACTED_KEY_]`, indexed over the sorted original key names so the response is stable across calls. This hides both the value and the key name, because either can carry a credential (`{"sk-...": 60}` is as much a leak as the reverse). + +| Map | Shown to a `read_only` caller | +|---|---| +| `plugins[].config` | only the keys that plugin's [catalog entry](/plugins) declares as settings — nothing for a plugin registered out of tree | +| `aliases` | everything — both sides are model names, resolved by the gateway itself | +| `mcp_servers[].env`, `mcp_servers[].headers` | nothing | +| `observability.exporters[].config`, `observability.tracing.headers` | nothing | + +A `${VAR}` reference inside a withheld entry's value is still served as written — it names a value rather than carrying one. + +`PUT`/`POST /admin/config` decode as strictly as a config file and refuse a body carrying a redaction placeholder (`[REDACTED]`, `[REDACTED_KEY_]`, or any other `[REDACTED...]` marker the read path can emit) anywhere in it, so an edit-and-resend of a `GET` response can never overwrite a live credential with placeholder text. Fields inside a withheld map have to be edited in the config file — a `GET` body with those keys removed cannot be edited and sent back. + +## Response cache is scoped per credential + +The `response-cache` plugin keys each cached entry on the request content **and** the caller's opaque `api_key_id` — never on content alone. Without that, a process-global cache would let a response primed by one credential be served to a different one, skipping the guardrails, rate limit, and budget checks that credential's own call would have triggered. Unauthenticated requests (when `ALLOW_UNAUTHENTICATED_PROXY=true`) share one bucket among themselves and never share with an authenticated caller. See [Response cache](/plugins/response-cache) for TTL and capacity config. + +## Audit trail + +Sign-ins (accepted and denied), credential changes, and log purges are written to a durable `audit_log` table and logged. The write is best-effort and never blocks or fails the action it records — a down audit store must not break key management, so on a persistently failing store the application log line is the record of what happened. + +`GET /admin/audit` (requires `read_only` or `admin` scope) reads the trail back, filterable by `action`, `actor_id`, `outcome`, and `since`, with `limit`/`offset` paging: + +```bash +curl "http://localhost:8080/admin/audit?action=key.create&since=2026-08-01T00:00:00Z" \ + -H "Authorization: Bearer $MASTER_KEY" +``` + +Each entry carries `occurred_at`, `action`, `actor`/`actor_id`, `target_id`, `outcome`, an optional redacted `detail`, `source_ip`, and a `trace_id` tying it to the request logs and OTel span for the same call. The store follows the API key store's backend — the in-memory default keeps only recent entries and does not survive a restart, so a deployment that needs full history configures a SQL backend. + +## Proxy path traversal refusal + +The `/v1/*` pass-through proxy, and the `/v1/files*`/`/v1/batches*` and `/v1/responses/*` surfaces built on the same forwarder, reject a request path containing a disallowed traversal segment (`400 invalid_proxy_path`) **before** the provider's credential is installed on the outbound request. Go preserves dot-segments (`..`) in `URL.Path` by design, and forwarding one unexamined could let a crafted path escape the configured `_BASE_URL` and reach an unintended upstream path carrying the gateway's credential. + ## Least privilege -- Use dedicated credentials for each provider. -- Restrict database permissions to only the required tables. -- Prefer TLS for external database connections. +- Use dedicated credentials for each provider, and a separate admin-scoped key per operator (see [Authentication](/guides/auth)) rather than sharing `MASTER_KEY` day to day. +- Restrict database permissions for the config, key, and request-log stores to only the required tables. +- Prefer TLS for external database connections (`CONFIG_STORE_DSN`, `API_KEY_STORE_DSN`, `REQUEST_LOG_STORE_DSN`). +- Keep `/metrics` and `/debug/*` off the public internet by deployment. `/metrics` needs `read_only` or `admin` scope; everything under `/debug` (`/debug/pprof/*`, `/debug/vars`) needs `admin`, since a profile can capture request bodies and credentials in memory and expvar publishes the process command line. + +## Related + +- [Authentication](/guides/auth) +- [MCP](/guides/mcp) +- [Response cache](/plugins/response-cache) +- [Admin API reference](/api-reference/admin) +- [Request logging](/operations/request-logging) diff --git a/docusaurus.config.ts b/docusaurus.config.ts index 58297d9..bc4c154 100644 --- a/docusaurus.config.ts +++ b/docusaurus.config.ts @@ -40,8 +40,13 @@ const structuredData = { name: 'Ferro Labs AI Gateway', applicationCategory: 'DeveloperApplication', operatingSystem: 'Linux, macOS, Windows, Docker, Kubernetes', + softwareVersion: '1.4.1', + // Folded in from the former homepage-local block, so the site keeps ONE + // SoftwareApplication entity instead of two conflicting copies. + license: 'https://opensource.org/licenses/Apache-2.0', + sameAs: ['https://github.com/ferro-labs/ai-gateway'], description: - 'Open-source, high-performance AI gateway written in Go. Routes LLM requests across 30 providers and 2,500+ models through a single OpenAI-compatible API, with 6 built-in plugins and 8 routing strategies.', + 'Open-source, high-performance AI gateway written in Go. Routes LLM requests across 30 providers and 2,500+ models through a single OpenAI-compatible API, with 6 built-in plugins, 8 routing strategies, MCP tool-calling, and an embedded dashboard.', url: SITE_URL, offers: { '@type': 'Offer', price: '0', priceCurrency: 'USD' }, publisher: { '@id': `${SITE_URL}/#organization` }, @@ -52,6 +57,15 @@ const structuredData = { url: SITE_URL, name: 'Ferro Labs AI Gateway Docs', publisher: { '@id': `${SITE_URL}/#organization` }, + // Sitelinks Search Box: /search/?q= is a real, working endpoint. + potentialAction: { + '@type': 'SearchAction', + target: { + '@type': 'EntryPoint', + urlTemplate: `${SITE_URL}/search/?q={search_term_string}`, + }, + 'query-input': 'required name=search_term_string', + }, }, ], }; @@ -101,18 +115,52 @@ const config: Config = { }, headTags: [ - // Preload the variable font so it paints with the first frame — removes the - // flash-of-unstyled-text and the layout shift it caused (better LCP/CLS). + // Preload the two variable fonts so they paint with the first frame — + // removes the flash-of-unstyled-text and the layout shift it caused + // (better LCP/CLS). Inter for prose/UI, JetBrains Mono for code. { tagName: 'link', attributes: { rel: 'preload', - href: '/fonts/roobert-proportional-vf.woff2', + href: '/fonts/inter-variable-latin.woff2', as: 'font', type: 'font/woff2', crossorigin: 'anonymous', }, }, + { + tagName: 'link', + attributes: { + rel: 'preload', + href: '/fonts/jetbrains-mono-variable-latin.woff2', + as: 'font', + type: 'font/woff2', + crossorigin: 'anonymous', + }, + }, + // @font-face lives here as a raw