Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion docs/benchmarking.md
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,16 @@ Add benchmarks only if they:

After adding a new benchmark, regenerate the baseline as described above.

### Dashboard

The Settings → Updates → Diagnostics panel reads from the runtime benchmark
database through `/api/benchmarks/*` endpoints. It shows:

- p50/p95 timing by stage for the selected window
- per-plugin timing averages
- recent refresh rows
- stage drill-down for a selected refresh

### Roadmap (next)

- Add `/api/benchmarks/*` endpoints and simple dashboard.
- SSE progress stream and lightweight UI indicator.
23 changes: 20 additions & 3 deletions src/blueprints/settings/_benchmarks.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,23 @@
from utils.messages import BENCHMARKS_API_DISABLED_ERROR


def _clamp_benchmark_limit(raw_limit: str | None) -> int:
try:
return max(1, min(200, int(raw_limit or "50")))
except (TypeError, ValueError):
return 50


def _parse_benchmark_cursor(raw_cursor: str | None) -> int | None:
if not raw_cursor:
return None
try:
cursor = int(raw_cursor)
except (TypeError, ValueError):
return None
return cursor if cursor > 0 else None


@_mod.settings_bp.route("/api/benchmarks/summary", methods=["GET"]) # type: ignore
def benchmarks_summary() -> tuple[object, int] | Response:
if not _mod._benchmarks_enabled():
Expand Down Expand Up @@ -66,8 +83,8 @@ def benchmarks_refreshes() -> tuple[object, int] | Response:
return json_error(BENCHMARKS_API_DISABLED_ERROR, status=404)
conn = None
try:
limit = max(1, min(200, int(request.args.get("limit", "50"))))
cursor = request.args.get("cursor")
limit = _clamp_benchmark_limit(request.args.get("limit"))
cursor = _parse_benchmark_cursor(request.args.get("cursor"))
since = _mod._window_since_seconds(request.args.get("window", "24h"))
conn = sqlite3.connect(_mod._get_bench_db_path())
conn.row_factory = sqlite3.Row
Expand All @@ -82,7 +99,7 @@ def benchmarks_refreshes() -> tuple[object, int] | Response:
ORDER BY id DESC
LIMIT ?
""",
(since, int(cursor), limit),
(since, cursor, limit),
).fetchall()
else:
rows = conn.execute(
Expand Down
4 changes: 4 additions & 0 deletions src/schemas/endpoint_map.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@

from schemas.responses import (
BenchmarksPluginsResponse,
BenchmarksRefreshesResponse,
BenchmarksStagesResponse,
BenchmarksSummaryResponse,
DiagnosticsResponse,
HealthPluginsResponse,
Expand Down Expand Up @@ -54,6 +56,8 @@
"settings.start_rollback": RollbackControlResponse,
"settings.benchmarks_summary": BenchmarksSummaryResponse,
"settings.benchmarks_plugins": BenchmarksPluginsResponse,
"settings.benchmarks_refreshes": BenchmarksRefreshesResponse,
"settings.benchmarks_stages": BenchmarksStagesResponse,
"settings.safe_reset": SuccessMessageResponse,
"settings.save_api_keys": SaveApiKeysResponse,
"settings.delete_api_key": SuccessMessageResponse,
Expand Down
43 changes: 43 additions & 0 deletions src/schemas/responses.py
Original file line number Diff line number Diff line change
Expand Up @@ -310,6 +310,49 @@ class BenchmarksPluginsResponse(TypedDict, total=False):
items: list[BenchmarksPluginEntry]


class BenchmarksRefreshEntry(TypedDict):
"""One recent refresh row from ``GET /api/benchmarks/refreshes``."""

id: int
ts: float
refresh_id: str
plugin_id: str | None
instance: str | None
playlist: str | None
used_cached: int | None
request_ms: int | None
generate_ms: int | None
preprocess_ms: int | None
display_ms: int | None


class BenchmarksRefreshesResponse(TypedDict, total=False):
"""Response body for ``GET /api/benchmarks/refreshes``."""

success: bool
request_id: str
items: list[BenchmarksRefreshEntry]
next_cursor: str | None


class BenchmarksStageEntry(TypedDict):
"""One stage event from ``GET /api/benchmarks/stages``."""

id: int
ts: float
stage: str
duration_ms: int | None
extra_json: str | None


class BenchmarksStagesResponse(TypedDict, total=False):
"""Response body for ``GET /api/benchmarks/stages``."""

success: bool
request_id: str
items: list[BenchmarksStageEntry]


class JobStatusResult(TypedDict, total=False):
"""Async ``/update_now`` job result payload returned by ``/api/job/<job_id>``."""

Expand Down
232 changes: 232 additions & 0 deletions src/static/openapi.json
Original file line number Diff line number Diff line change
Expand Up @@ -650,6 +650,103 @@
}
}
},
"/api/benchmarks/refreshes": {
"get": {
"tags": [
"Health"
],
"summary": "Recent benchmark refreshes",
"description": "Returns recent refresh timing rows for the diagnostics dashboard.",
"operationId": "benchmarksRefreshes",
"parameters": [
{
"name": "window",
"in": "query",
"required": false,
"schema": {
"type": "string",
"default": "24h"
},
"description": "Look-back window (for example 1h, 24h, 7d)."
},
{
"name": "limit",
"in": "query",
"required": false,
"schema": {
"type": "integer",
"minimum": 1,
"maximum": 200,
"default": 50
},
"description": "Maximum rows to return."
},
{
"name": "cursor",
"in": "query",
"required": false,
"schema": {
"type": "string"
},
"description": "Previous refresh_events id for pagination."
}
],
"responses": {
"200": {
"description": "Recent benchmark refresh rows",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/BenchmarksRefreshesResponse"
}
}
}
},
"404": {
"$ref": "#/components/responses/NotFound"
}
}
}
},
"/api/benchmarks/stages": {
"get": {
"tags": [
"Health"
],
"summary": "Benchmark stage events",
"description": "Returns stage timing rows for a single refresh_id.",
"operationId": "benchmarksStages",
"parameters": [
{
"name": "refresh_id",
"in": "query",
"required": true,
"schema": {
"type": "string"
},
"description": "Refresh id from /api/benchmarks/refreshes."
}
],
"responses": {
"200": {
"description": "Benchmark stage rows",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/BenchmarksStagesResponse"
}
}
}
},
"404": {
"$ref": "#/components/responses/NotFound"
},
"422": {
"$ref": "#/components/responses/BadRequest"
}
}
}
},
"/api/diagnostics": {
"get": {
"tags": [
Expand Down Expand Up @@ -2096,6 +2193,141 @@
}
}
},
"BenchmarksRefreshEntry": {
"type": "object",
"required": [
"id",
"ts",
"refresh_id",
"plugin_id",
"instance",
"playlist",
"used_cached",
"request_ms",
"generate_ms",
"preprocess_ms",
"display_ms"
],
"properties": {
"id": {
"type": "integer"
},
"ts": {
"type": "number"
},
"refresh_id": {
"type": "string"
},
"plugin_id": {
"type": "string",
"nullable": true
},
"instance": {
"type": "string",
"nullable": true
},
"playlist": {
"type": "string",
"nullable": true
},
"used_cached": {
"type": "integer",
"nullable": true
},
"request_ms": {
"type": "integer",
"nullable": true
},
"generate_ms": {
"type": "integer",
"nullable": true
},
"preprocess_ms": {
"type": "integer",
"nullable": true
},
"display_ms": {
"type": "integer",
"nullable": true
}
}
},
"BenchmarksRefreshesResponse": {
"type": "object",
"required": [
"success",
"items",
"next_cursor"
],
"properties": {
"success": {
"type": "boolean"
},
"request_id": {
"type": "string"
},
"items": {
"type": "array",
"items": {
"$ref": "#/components/schemas/BenchmarksRefreshEntry"
}
},
"next_cursor": {
"type": "string",
"nullable": true
}
}
},
"BenchmarksStageEntry": {
"type": "object",
"required": [
"id",
"ts",
"stage",
"duration_ms",
"extra_json"
],
"properties": {
"id": {
"type": "integer"
},
"ts": {
"type": "number"
},
"stage": {
"type": "string"
},
"duration_ms": {
"type": "integer",
"nullable": true
},
"extra_json": {
"type": "string",
"nullable": true
}
}
},
"BenchmarksStagesResponse": {
"type": "object",
"required": [
"success",
"items"
],
"properties": {
"success": {
"type": "boolean"
},
"request_id": {
"type": "string"
},
"items": {
"type": "array",
"items": {
"$ref": "#/components/schemas/BenchmarksStageEntry"
}
}
}
},
"DiagnosticsMemory": {
"type": "object",
"required": [
Expand Down
Loading
Loading