diff --git a/data/canary-runs.json b/data/canary-runs.json index 8932c426..1c0bea16 100644 --- a/data/canary-runs.json +++ b/data/canary-runs.json @@ -1,3 +1,272 @@ -{ - "results": [] -} +[ + { + "id": "run_001", + "skillSlug": "stripe-subscription-checkout", + "skillName": "Stripe Subscription Checkout", + "version": "2.4.1", + "status": "pass", + "duration": 1240, + "timestamp": "2026-02-09T18:02:11.000Z", + "checks": [ + { + "name": "checkout_session_created", + "status": "pass", + "message": "Created Checkout Session with valid customer and trial settings." + }, + { + "name": "webhook_signature_verified", + "status": "pass", + "message": "Stripe webhook signature validated and event persisted." + }, + { + "name": "subscription_state_synced", + "status": "pass", + "message": "Subscription status reflected in tenant billing profile." + } + ], + "regression": false + }, + { + "id": "run_002", + "skillSlug": "supabase-rls-audit", + "skillName": "Supabase RLS Audit", + "version": "1.9.0", + "status": "warning", + "duration": 2310, + "timestamp": "2026-02-09T17:41:37.000Z", + "checks": [ + { + "name": "anonymous_reads_blocked", + "status": "pass", + "message": "Anonymous role cannot access private tenant rows." + }, + { + "name": "service_role_write_scope", + "status": "warning", + "message": "Service role can still write to archival table; review policy scope." + }, + { + "name": "cross_tenant_isolation", + "status": "pass", + "message": "Cross-tenant reads denied for authenticated users." + } + ], + "regression": true + }, + { + "id": "run_003", + "skillSlug": "nextjs-cache-headers", + "skillName": "Next.js Cache Headers", + "version": "3.1.2", + "status": "pass", + "duration": 980, + "timestamp": "2026-02-09T17:16:08.000Z", + "checks": [ + { + "name": "etag_present", + "status": "pass", + "message": "ETag header present on static content routes." + }, + { + "name": "api_no_store", + "status": "pass", + "message": "Dynamic API responses include no-store cache control." + }, + { + "name": "stale_while_revalidate", + "status": "pass", + "message": "Public feed routes return SWR directives as expected." + } + ], + "regression": false + }, + { + "id": "run_004", + "skillSlug": "agent-bootstrap-health", + "skillName": "Agent Bootstrap Health", + "version": "4.0.0", + "status": "fail", + "duration": 3011, + "timestamp": "2026-02-09T16:53:40.000Z", + "checks": [ + { + "name": "bootstrap_json_available", + "status": "pass", + "message": "Bootstrap JSON endpoint returned 200 with valid schema." + }, + { + "name": "bootstrap_markdown_integrity", + "status": "fail", + "message": "Markdown route missing required section: ## Memory." + }, + { + "name": "quickstart_route", + "status": "pass", + "message": "Quickstart route rendered expected onboarding blocks." + } + ], + "regression": true + }, + { + "id": "run_005", + "skillSlug": "mcp-server-catalog", + "skillName": "MCP Server Catalog", + "version": "2.2.0", + "status": "pass", + "duration": 1678, + "timestamp": "2026-02-09T16:20:02.000Z", + "checks": [ + { + "name": "catalog_endpoint", + "status": "pass", + "message": "MCP server catalog endpoint returned complete listing." + }, + { + "name": "install_links_valid", + "status": "pass", + "message": "All install links resolved with 2xx responses." + }, + { + "name": "slug_uniqueness", + "status": "pass", + "message": "No duplicate server slugs detected." + } + ], + "regression": false + }, + { + "id": "run_006", + "skillSlug": "stripe-subscription-checkout", + "skillName": "Stripe Subscription Checkout", + "version": "2.4.0", + "status": "warning", + "duration": 2144, + "timestamp": "2026-02-09T15:49:59.000Z", + "checks": [ + { + "name": "checkout_session_created", + "status": "pass", + "message": "Checkout session generated and returned hosted URL." + }, + { + "name": "customer_portal_link", + "status": "warning", + "message": "Customer portal deep link returned intermittent 429 responses." + }, + { + "name": "subscription_state_synced", + "status": "pass", + "message": "Subscription status persisted to billing table." + } + ], + "regression": false + }, + { + "id": "run_007", + "skillSlug": "supabase-rls-audit", + "skillName": "Supabase RLS Audit", + "version": "1.8.9", + "status": "pass", + "duration": 1895, + "timestamp": "2026-02-09T15:18:34.000Z", + "checks": [ + { + "name": "anonymous_reads_blocked", + "status": "pass", + "message": "Anonymous reads correctly blocked for protected tables." + }, + { + "name": "service_role_write_scope", + "status": "pass", + "message": "Service role writes constrained to designated tables only." + }, + { + "name": "cross_tenant_isolation", + "status": "pass", + "message": "Tenant-scoped claims prevent data leakage between orgs." + } + ], + "regression": false + }, + { + "id": "run_008", + "skillSlug": "agent-bootstrap-health", + "skillName": "Agent Bootstrap Health", + "version": "3.9.8", + "status": "pass", + "duration": 1422, + "timestamp": "2026-02-09T14:44:10.000Z", + "checks": [ + { + "name": "bootstrap_json_available", + "status": "pass", + "message": "Bootstrap JSON available and valid." + }, + { + "name": "bootstrap_markdown_integrity", + "status": "pass", + "message": "Markdown includes all required sections and examples." + }, + { + "name": "quickstart_route", + "status": "pass", + "message": "Quickstart endpoint delivered expected response body." + } + ], + "regression": false + }, + { + "id": "run_009", + "skillSlug": "search-index-freshness", + "skillName": "Search Index Freshness", + "version": "1.3.5", + "status": "fail", + "duration": 2777, + "timestamp": "2026-02-09T14:11:26.000Z", + "checks": [ + { + "name": "index_latency", + "status": "warning", + "message": "Index lag exceeded target by 12 minutes." + }, + { + "name": "document_count_match", + "status": "fail", + "message": "Indexed document count is behind source by 247 records." + }, + { + "name": "query_relevancy_smoke", + "status": "pass", + "message": "Top 3 results for smoke queries remained stable." + } + ], + "regression": true + }, + { + "id": "run_010", + "skillSlug": "webhook-retry-monitor", + "skillName": "Webhook Retry Monitor", + "version": "0.8.4", + "status": "pass", + "duration": 1328, + "timestamp": "2026-02-09T13:36:51.000Z", + "checks": [ + { + "name": "retry_queue_depth", + "status": "pass", + "message": "Retry queue depth stayed below alert threshold." + }, + { + "name": "dead_letter_routing", + "status": "pass", + "message": "Failed events routed to dead-letter topic correctly." + }, + { + "name": "backoff_schedule", + "status": "pass", + "message": "Exponential backoff intervals matched policy settings." + } + ], + "regression": false + } +] diff --git a/src/app/api/canary/[id]/route.ts b/src/app/api/canary/[id]/route.ts new file mode 100644 index 00000000..7593486b --- /dev/null +++ b/src/app/api/canary/[id]/route.ts @@ -0,0 +1,30 @@ +import { NextRequest, NextResponse } from "next/server"; +import { readCanaryRuns } from "@/lib/server/canaryRunsStore"; + +export const runtime = "nodejs"; +export const dynamic = "force-dynamic"; + +export async function GET(_request: NextRequest, { params }: { params: Promise<{ id: string }> }) { + const { id } = await params; + const runId = typeof id === "string" ? id.trim() : ""; + + if (!runId) { + return NextResponse.json({ error: "id is required" }, { status: 400 }); + } + + const runs = await readCanaryRuns(); + const run = runs.find((entry) => entry.id === runId); + + if (!run) { + return NextResponse.json({ error: "Canary run not found" }, { status: 404 }); + } + + return NextResponse.json( + { run }, + { + headers: { + "Cache-Control": "no-store", + }, + } + ); +} diff --git a/src/app/api/canary/route.ts b/src/app/api/canary/route.ts new file mode 100644 index 00000000..efc62c1d --- /dev/null +++ b/src/app/api/canary/route.ts @@ -0,0 +1,140 @@ +import { NextRequest, NextResponse } from "next/server"; +import { readJsonWithLimit } from "@/lib/requestLimits"; +import type { CanaryCheck, CanaryRunStatus } from "@/lib/canaryRuns"; +import { appendCanaryRun, readCanaryRuns } from "@/lib/server/canaryRunsStore"; + +export const runtime = "nodejs"; +export const dynamic = "force-dynamic"; + +const MAX_BODY_BYTES = 32_000; + +function isStatus(value: unknown): value is CanaryRunStatus { + return value === "pass" || value === "fail" || value === "warning"; +} + +function sanitizePage(raw: string | null): number { + const page = raw ? Number.parseInt(raw, 10) : 1; + if (!Number.isFinite(page) || page < 1) return 1; + return page; +} + +function sanitizePageSize(raw: string | null): number { + const pageSize = raw ? Number.parseInt(raw, 10) : 10; + if (!Number.isFinite(pageSize)) return 10; + return Math.max(1, Math.min(100, pageSize)); +} + +function normalizeCheckInput(raw: unknown): CanaryCheck | null { + if (!raw || typeof raw !== "object") return null; + + const obj = raw as Record; + const name = typeof obj.name === "string" ? obj.name.trim() : ""; + const message = typeof obj.message === "string" ? obj.message.trim() : ""; + + if (!name || !message || !isStatus(obj.status)) return null; + + return { + name, + status: obj.status, + message, + }; +} + +export async function GET(request: NextRequest) { + const searchParams = request.nextUrl.searchParams; + const status = searchParams.get("status"); + const skill = searchParams.get("skill")?.trim().toLowerCase() ?? ""; + const page = sanitizePage(searchParams.get("page")); + const pageSize = sanitizePageSize(searchParams.get("pageSize")); + + const allRuns = await readCanaryRuns(); + + const filtered = allRuns.filter((run) => { + const statusMatch = isStatus(status) ? run.status === status : true; + const skillMatch = skill + ? run.skillSlug.toLowerCase().includes(skill) || run.skillName.toLowerCase().includes(skill) + : true; + + return statusMatch && skillMatch; + }); + + const total = filtered.length; + const totalPages = Math.max(1, Math.ceil(total / pageSize)); + const safePage = Math.min(page, totalPages); + const start = (safePage - 1) * pageSize; + const runs = filtered.slice(start, start + pageSize); + + return NextResponse.json( + { + runs, + pagination: { + page: safePage, + pageSize, + total, + totalPages, + hasNextPage: safePage < totalPages, + hasPreviousPage: safePage > 1, + }, + filters: { + status: isStatus(status) ? status : null, + skill: skill || null, + }, + skills: Array.from(new Set(allRuns.map((run) => run.skillSlug))).sort((a, b) => a.localeCompare(b)), + }, + { + headers: { + "Cache-Control": "no-store", + }, + } + ); +} + +export async function POST(request: NextRequest) { + try { + const body = await readJsonWithLimit>(request, MAX_BODY_BYTES); + + const skillSlug = typeof body.skillSlug === "string" ? body.skillSlug.trim() : ""; + const skillName = typeof body.skillName === "string" ? body.skillName.trim() : ""; + const version = typeof body.version === "string" ? body.version.trim() : ""; + const duration = typeof body.duration === "number" ? body.duration : NaN; + const rawStatus = body.status; + const status = isStatus(rawStatus) ? rawStatus : null; + + const checks = Array.isArray(body.checks) + ? body.checks + .map((item) => normalizeCheckInput(item)) + .filter((item): item is CanaryCheck => Boolean(item)) + : []; + + const errors: string[] = []; + + if (!skillSlug) errors.push("skillSlug is required"); + if (!skillName) errors.push("skillName is required"); + if (!version) errors.push("version is required"); + if (!status) errors.push("status must be one of: pass, fail, warning"); + if (!Number.isFinite(duration) || duration < 0) errors.push("duration must be a non-negative number"); + if (checks.length === 0) errors.push("checks must include at least one valid check"); + + if (errors.length > 0) { + return NextResponse.json({ error: "Validation failed", details: errors }, { status: 400 }); + } + + const created = await appendCanaryRun({ + skillSlug, + skillName, + version, + status: status as CanaryRunStatus, + duration, + checks, + }); + + return NextResponse.json( + { + run: created, + }, + { status: 201 } + ); + } catch { + return NextResponse.json({ error: "Invalid request body. Expected JSON." }, { status: 400 }); + } +} diff --git a/src/app/canary/page.tsx b/src/app/canary/page.tsx index d3e8e1cc..50ec4aa3 100644 --- a/src/app/canary/page.tsx +++ b/src/app/canary/page.tsx @@ -1,9 +1,11 @@ +/* eslint-disable react/no-unescaped-entities */ import type { Metadata } from "next"; import Link from "next/link"; -import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { Badge } from "@/components/ui/badge"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { Separator } from "@/components/ui/separator"; -import { readCanaryResults } from "@/lib/server/canaryStore"; +import type { CanaryRunStatus } from "@/lib/canaryRuns"; +import { readCanaryRuns } from "@/lib/server/canaryRunsStore"; export const dynamic = "force-dynamic"; @@ -33,82 +35,132 @@ export const metadata: Metadata = { }, }; -function formatMs(ms: number) { +type CanaryPageProps = { + searchParams?: Promise<{ + status?: string; + skill?: string; + page?: string; + pageSize?: string; + }>; +}; + +function formatDuration(ms: number) { if (!Number.isFinite(ms)) return "—"; if (ms < 1000) return `${Math.round(ms)}ms`; return `${(ms / 1000).toFixed(2)}s`; } -export default async function CanaryPage() { - const results = await readCanaryResults(); - const recent = results.slice(0, 100); - - const total = recent.length; - const passes = recent.filter((r) => r.status === "pass").length; - const uptimePct = total ? (passes / total) * 100 : 0; - const avgResponseMs = - total > 0 - ? recent.reduce((sum, r) => sum + (Number.isFinite(r.responseTimeMs) ? r.responseTimeMs : 0), 0) / - total - : 0; +function isStatus(value: unknown): value is CanaryRunStatus { + return value === "pass" || value === "fail" || value === "warning"; +} + +function statusClass(status: CanaryRunStatus) { + if (status === "pass") return "bg-green-500/10 text-green-400 border-green-500/30"; + if (status === "warning") return "bg-yellow-500/10 text-yellow-400 border-yellow-500/30"; + return "bg-red-500/10 text-red-400 border-red-500/30"; +} + +function sanitizePositiveInt(raw: string | undefined, fallback: number) { + const parsed = raw ? Number.parseInt(raw, 10) : fallback; + if (!Number.isFinite(parsed) || parsed < 1) return fallback; + return parsed; +} + +function buildPageHref({ + status, + skill, + page, + pageSize, +}: { + status: string; + skill: string; + page: number; + pageSize: number; +}) { + const params = new URLSearchParams(); + if (status) params.set("status", status); + if (skill) params.set("skill", skill); + params.set("page", String(page)); + params.set("pageSize", String(pageSize)); + return `/canary?${params.toString()}`; +} + +export default async function CanaryPage({ searchParams }: CanaryPageProps) { + const sp = (await searchParams) || {}; + const selectedStatus = isStatus(sp.status) ? sp.status : ""; + const selectedSkill = (sp.skill ?? "").trim().toLowerCase(); + const page = sanitizePositiveInt(sp.page, 1); + const pageSize = Math.min(50, sanitizePositiveInt(sp.pageSize, 8)); - const lastFailure = recent.find((r) => r.status === "fail"); + const allRuns = await readCanaryRuns(); + + const filteredRuns = allRuns.filter((run) => { + const statusMatch = selectedStatus ? run.status === selectedStatus : true; + const skillMatch = selectedSkill + ? run.skillSlug.toLowerCase().includes(selectedSkill) || run.skillName.toLowerCase().includes(selectedSkill) + : true; + + return statusMatch && skillMatch; + }); + + const totalRuns = filteredRuns.length; + const totalPages = Math.max(1, Math.ceil(totalRuns / pageSize)); + const safePage = Math.min(page, totalPages); + const start = (safePage - 1) * pageSize; + const paginatedRuns = filteredRuns.slice(start, start + pageSize); + + const uniqueSkills = Array.from(new Set(allRuns.map((run) => run.skillSlug))).sort((a, b) => a.localeCompare(b)); + + const passCount = filteredRuns.filter((run) => run.status === "pass").length; + const warningCount = filteredRuns.filter((run) => run.status === "warning").length; + const failCount = filteredRuns.filter((run) => run.status === "fail").length; + const avgDuration = + filteredRuns.length > 0 + ? filteredRuns.reduce((sum, run) => sum + run.duration, 0) / filteredRuns.length + : 0; return (
- -
+

Canary Runs

-

- Lightweight regression checks against key endpoints. -

+

Persistent canary history with skill-level regression detection.

-
+
- Uptime (last 100 checks) + Passing -
- {uptimePct.toFixed(2)}% -
-
- {passes}/{total} passing -
+
{passCount}
 - Avg Response Time + Warnings -
- {formatMs(avgResponseMs)} -
-
Across last 100 checks
+
{warningCount}
 - Last Failure + Failures - {lastFailure ? ( -
-
- {lastFailure.endpoint} -
-
- {new Date(lastFailure.checkedAt).toLocaleString()} · {lastFailure.errorMessage ?? "Failed"} -
-
- ) : ( -
No failures in last 100 checks.
- )} +
{failCount}
+ + + + + + Avg Duration + + +
{formatDuration(avgDuration)}
@@ -116,87 +168,167 @@ export default async function CanaryPage() { -
-
+
+
-

Recent Results

-

- Each row is a single endpoint check. -

+

Timeline

+

Filter by status/skill, then expand a run to inspect individual checks.

- - View JSON → + + View API JSON →
- - -
- - - - - - - - - - - - {recent.length === 0 ? ( - - - - ) : ( - recent.map((r) => ( - - - - - - - - )) - )} - -
TimestampEndpointStatusResponseError
- No canary results yet. Run POST /api/canary/run to generate results. -
- {new Date(r.checkedAt).toLocaleString()} - - {r.endpoint} - -
- - {r.status === "pass" ? "PASS" : "FAIL"} - - {r.regression ? ( - - REGRESSION - - ) : null} -
- 		- {formatMs(r.responseTimeMs)} - - {r.errorMessage ?? "—"} -
-
+ + +
+ + + + + + +
+ + + Reset + +
+
+ + {paginatedRuns.length === 0 ? ( + + No runs match the active filters. + + ) : ( +
+ {paginatedRuns.map((run, index) => ( +
+ {index !== paginatedRuns.length - 1 ? ( + + ) : null} + + + + +
+
+
{new Date(run.timestamp).toLocaleString()}
+
{run.skillName}
+
{run.skillSlug} · v{run.version}
+
+ +
+ + {run.status.toUpperCase()} + + {run.regression ? ( + + REGRESSION + + ) : null} + + {formatDuration(run.duration)} + +
+
+ +
+ View check details +
+ {run.checks.map((check) => ( +
+
+
{check.name}
+ + {check.status.toUpperCase()} + +
+

{check.message}

+
+ ))} +
+
+ + +
+ ))} +
+ )} + +
+ + Showing page {safePage} of {totalPages} · {totalRuns} run{totalRuns === 1 ? "" : "s"} + + +
+ {safePage > 1 ? ( + + Previous + + ) : null} + + {safePage < totalPages ? ( + + Next + + ) : null} +
+
); diff --git a/src/lib/canaryRuns.ts b/src/lib/canaryRuns.ts new file mode 100644 index 00000000..1e978c8e --- /dev/null +++ b/src/lib/canaryRuns.ts @@ -0,0 +1,19 @@ +export type CanaryRunStatus = "pass" | "fail" | "warning"; + +export interface CanaryCheck { + name: string; + status: CanaryRunStatus; + message: string; +} + +export interface CanaryRun { + id: string; + skillSlug: string; + skillName: string; + version: string; + status: CanaryRunStatus; + duration: number; + timestamp: string; + checks: CanaryCheck[]; + regression: boolean; +} diff --git a/src/lib/server/canaryRunsStore.ts b/src/lib/server/canaryRunsStore.ts new file mode 100644 index 00000000..ba620573 --- /dev/null +++ b/src/lib/server/canaryRunsStore.ts @@ -0,0 +1,110 @@ +import { promises as fs } from "fs"; +import path from "path"; +import type { CanaryCheck, CanaryRun, CanaryRunStatus } from "@/lib/canaryRuns"; + +const CANARY_RUNS_PATH = path.join(process.cwd(), "data", "canary-runs.json"); +const MAX_STORED_RUNS = 1_000; + +function isStatus(value: unknown): value is CanaryRunStatus { + return value === "pass" || value === "fail" || value === "warning"; +} + +function normalizeCheck(raw: unknown): CanaryCheck | null { + if (!raw || typeof raw !== "object") return null; + + const obj = raw as Record; + const name = typeof obj.name === "string" ? obj.name.trim() : ""; + const message = typeof obj.message === "string" ? obj.message.trim() : ""; + + if (!name || !message || !isStatus(obj.status)) return null; + + return { + name, + status: obj.status, + message, + }; +} + +function normalizeRun(raw: unknown): CanaryRun | null { + if (!raw || typeof raw !== "object") return null; + + const obj = raw as Record; + const checks = Array.isArray(obj.checks) + ? obj.checks.map((item) => normalizeCheck(item)).filter((item): item is CanaryCheck => Boolean(item)) + : []; + + if ( + typeof obj.id !== "string" || + typeof obj.skillSlug !== "string" || + typeof obj.skillName !== "string" || + typeof obj.version !== "string" || + !isStatus(obj.status) || + typeof obj.duration !== "number" || + !Number.isFinite(obj.duration) || + obj.duration < 0 || + typeof obj.timestamp !== "string" || + typeof obj.regression !== "boolean" || + checks.length === 0 + ) { + return null; + } + + return { + id: obj.id, + skillSlug: obj.skillSlug, + skillName: obj.skillName, + version: obj.version, + status: obj.status, + duration: obj.duration, + timestamp: obj.timestamp, + checks, + regression: obj.regression, + }; +} + +function sortRunsDesc(runs: CanaryRun[]): CanaryRun[] { + return [...runs].sort((a, b) => b.timestamp.localeCompare(a.timestamp)); +} + +export async function readCanaryRuns(): Promise { + try { + const raw = await fs.readFile(CANARY_RUNS_PATH, "utf-8"); + const parsed = JSON.parse(raw) as unknown; + + const list = Array.isArray(parsed) + ? parsed + : parsed && typeof parsed === "object" && Array.isArray((parsed as { runs?: unknown[] }).runs) + ? (parsed as { runs: unknown[] }).runs + : []; + + return sortRunsDesc( + list + .map((item) => normalizeRun(item)) + .filter((item): item is CanaryRun => Boolean(item)) + ); + } catch { + return []; + } +} + +export async function writeCanaryRuns(runs: CanaryRun[]): Promise { + await fs.mkdir(path.dirname(CANARY_RUNS_PATH), { recursive: true }); + const sorted = sortRunsDesc(runs).slice(0, MAX_STORED_RUNS); + await fs.writeFile(CANARY_RUNS_PATH, JSON.stringify(sorted, null, 2) + "\n", "utf-8"); +} + +export async function appendCanaryRun(input: Omit): Promise { + const existing = await readCanaryRuns(); + const previousForSkill = existing.find((run) => run.skillSlug === input.skillSlug); + const regression = previousForSkill?.status === "pass" && input.status !== "pass"; + + const newRun: CanaryRun = { + ...input, + id: globalThis.crypto?.randomUUID?.() ?? `canary_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`, + timestamp: new Date().toISOString(), + regression, + }; + + await writeCanaryRuns([newRun, ...existing]); + return newRun; +}