diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index 6554aa41..13691d13 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -302,16 +302,24 @@ jobs: env: EVAL_PAIRS: ${{ needs.prepare.outputs.pairs }} EVAL_REVISION: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + # Optional warm-boot snapshot to reuse across runs (built out-of-band). + # Unset -> cold boots, so this can never regress the default path. + EVAL_SNAPSHOT_ID: ${{ vars.EVAL_SNAPSHOT_ID }} shell: bash run: | set -euo pipefail - pnpm --filter @supabase-evals/framework eval:vercel -- \ - --pairs-json "$EVAL_PAIRS" \ - --revision "$EVAL_REVISION" \ - --runs "${{ needs.prepare.outputs.runs }}" \ - --timeout-sec "${{ needs.prepare.outputs.timeout_sec }}" \ + args=( + --pairs-json "$EVAL_PAIRS" + --revision "$EVAL_REVISION" + --runs "${{ needs.prepare.outputs.runs }}" + --timeout-sec "${{ needs.prepare.outputs.timeout_sec }}" --concurrency "${{ needs.prepare.outputs.sandbox_concurrency }}" + ) + if [ -n "${EVAL_SNAPSHOT_ID:-}" ]; then + args+=(--snapshot-id "$EVAL_SNAPSHOT_ID") + fi + pnpm --filter @supabase-evals/framework eval:vercel -- "${args[@]}" - name: Upload raw results if: always() diff --git a/apps/framework/scripts/run-vercel-evals.test.ts b/apps/framework/scripts/run-vercel-evals.test.ts index 326878f6..cc364a30 100644 --- a/apps/framework/scripts/run-vercel-evals.test.ts +++ b/apps/framework/scripts/run-vercel-evals.test.ts @@ -1,11 +1,7 @@ import { APIError } from '@vercel/sandbox'; import { describe, expect, it } from 'vitest'; -import { - isRetryableSandboxCreateError, - parsePairs, - runBounded, - tagValue, -} from './run-vercel-evals.js'; +import { parsePairs, runBounded } from './run-vercel-evals.js'; +import { isRetryableSandboxCreateError, tagValue } from './vercel-sandbox.js'; describe('Vercel eval controller', () => { it('bounds concurrent work and lets independent failures settle', async () => { diff --git a/apps/framework/scripts/run-vercel-evals.ts b/apps/framework/scripts/run-vercel-evals.ts index 5efe6a94..9f52393a 100644 --- a/apps/framework/scripts/run-vercel-evals.ts +++ b/apps/framework/scripts/run-vercel-evals.ts @@ -1,16 +1,29 @@ #!/usr/bin/env tsx -import { APIError, Sandbox } from '@vercel/sandbox'; import { execFile, execFileSync } from 'node:child_process'; import { mkdtempSync, mkdirSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { promisify } from 'node:util'; +import type { Sandbox } from '@vercel/sandbox'; import pLimit from 'p-limit'; import pRetry from 'p-retry'; import { z } from 'zod'; import { positiveInteger, readFlag } from '../lib/cli-args.js'; +import { + SANDBOX_CWD, + coldProvision, + createSandbox, + errorMessage, + installDependencies, + runSandboxCommand, + startDocker, + tagValue, + vercelCredentialsFromEnv, + type VercelCredentials, +} from './vercel-sandbox.js'; +import { ensureSnapshot } from './vercel-snapshot.js'; const ROOT = fileURLToPath(new URL('../../../', import.meta.url)); const execFileAsync = promisify(execFile); @@ -34,6 +47,18 @@ const EVAL_TIMEOUT_BUFFER_MS = 25 * 60 * 1_000; */ const SANDBOX_TIMEOUT_BUFFER_MS = 10 * 60 * 1_000; +/** + * Wall-clock budget for the `pnpm eval` command: every attempt at its full + * timeout plus buffer for the non-agent work around it. Shared by the sandbox + * session ceiling and the command timer so the two can't drift. + */ +function evalCommandTimeoutMs(options: { + runs: number; + timeoutSec: number; +}): number { + return options.runs * options.timeoutSec * 1_000 + EVAL_TIMEOUT_BUFFER_MS; +} + const evalPairSchema = z.object({ eval_id: z.string(), experiment: z.string(), @@ -51,6 +76,11 @@ interface RunnerOptions { timeoutSec: number; concurrency: number; vcpus: number; + /** + * Warm-boot snapshot to start each pair's VM from (docker + node_modules + + * Supabase images pre-baked). Undefined means a cold git-source boot. + */ + snapshotId?: string; } interface PairOptions extends RunnerOptions { @@ -58,23 +88,6 @@ interface PairOptions extends RunnerOptions { attempt: number; } -interface SandboxCommandOptions { - cmd: string; - args?: string[]; - cwd?: string; - env?: Record; - sudo?: boolean; - timeoutMs?: number; -} - -class SandboxCommandError extends Error { - constructor(step: string, exitCode: number, output: string) { - const detail = output.trim().slice(-4_000); - super(`${step} exited with code ${exitCode}${detail ? `\n${detail}` : ''}`); - this.name = 'SandboxCommandError'; - } -} - /** Runs every item while keeping at most `concurrency` promises active. */ export async function runBounded( items: readonly T[], @@ -159,21 +172,25 @@ async function runPairOnce( let sandbox: Sandbox | undefined; try { + const warm = Boolean(options.snapshotId); sandbox = await createSandbox(label, { ...credentials, name: sandboxName(pair), - runtime: 'node24', - source: { - type: 'git', - url: options.repoUrl, - revision: options.revision, - depth: 1, - }, + // A snapshot source carries its own runtime + filesystem; a git source + // needs the runtime named and clones the repo fresh. + ...(options.snapshotId + ? { source: { type: 'snapshot', snapshotId: options.snapshotId } } + : { + runtime: 'node24', + source: { + type: 'git', + url: options.repoUrl, + revision: options.revision, + depth: 1, + }, + }), resources: { vcpus: options.vcpus }, - timeout: - options.runs * options.timeoutSec * 1_000 + - EVAL_TIMEOUT_BUFFER_MS + - SANDBOX_TIMEOUT_BUFFER_MS, + timeout: evalCommandTimeoutMs(options) + SANDBOX_TIMEOUT_BUFFER_MS, persistent: false, tags: { runner: 'supabase-evals', @@ -184,60 +201,32 @@ async function runPairOnce( }, }); console.log( - `${label} attempt ${options.attempt}: ${SANDBOX_DASHBOARD_URL}/${sandbox.name}` + `${label} attempt ${options.attempt} (${warm ? 'warm' : 'cold'} boot): ${SANDBOX_DASHBOARD_URL}/${sandbox.name}` ); - await runSandboxCommand(sandbox, label, 'initialize submodules', { - cmd: 'git', - args: [ - '-c', - 'url.https://github.com/.insteadOf=git@github.com:', - 'submodule', - 'update', - '--init', - '--recursive', - ], - cwd: sandbox.cwd, - timeoutMs: 2 * 60 * 1_000, - }); - await runSandboxCommand(sandbox, label, 'install Docker', { - cmd: 'dnf', - args: ['install', '-y', '-q', 'docker'], - sudo: true, - timeoutMs: 3 * 60 * 1_000, - }); - await sandbox.runCommand({ - cmd: 'dockerd', - sudo: true, - detached: true, - }); - await runSandboxCommand(sandbox, label, 'start Docker', { - cmd: 'bash', - args: [ - '-c', - 'for i in $(seq 60); do docker info >/dev/null 2>&1 && chmod 666 /var/run/docker.sock && exit 0; sleep 1; done; echo "dockerd not ready" >&2; exit 1', - ], - sudo: true, - timeoutMs: 90_000, - }); - // Installs the pnpm version pinned by packageManager in the checked-out - // root package.json so the two can't drift. - await runSandboxCommand(sandbox, label, 'install pnpm', { - cmd: 'bash', - args: [ - '-c', - `npm install --global "$(node -p 'require("./package.json").packageManager')"`, - ], - cwd: sandbox.cwd, - sudo: true, - timeoutMs: 2 * 60 * 1_000, - }); - await runSandboxCommand(sandbox, label, 'install dependencies', { - cmd: 'pnpm', - args: ['install', '--frozen-lockfile'], - cwd: sandbox.cwd, - timeoutMs: 6 * 60 * 1_000, - }); + if (warm) { + // The snapshot already carries docker, node_modules, and the pulled + // images; only restart the daemon (processes aren't snapshotted), move + // the checkout to the target revision, and reconcile dependency drift. + await startDocker(sandbox, label, { install: false }); + await runSandboxCommand(sandbox, label, 'checkout revision', { + cmd: 'bash', + args: [ + '-c', + // supabase/evals is public, so no credentials are needed to fetch. + 'git config --global url."https://github.com/".insteadOf "git@github.com:"\n' + + 'git fetch --quiet --depth 1 origin "$REVISION"\n' + + 'git checkout --quiet --force "$REVISION"\n' + + 'git submodule update --init --recursive', + ], + cwd: SANDBOX_CWD, + env: { REVISION: options.revision }, + timeoutMs: 3 * 60 * 1_000, + }); + await installDependencies(sandbox, label); + } else { + await coldProvision(sandbox, label); + } await sandbox.writeFiles([ { @@ -266,16 +255,15 @@ async function runPairOnce( '--timeout-sec', String(options.timeoutSec), ], - cwd: sandbox.cwd, - timeoutMs: - options.runs * options.timeoutSec * 1_000 + EVAL_TIMEOUT_BUFFER_MS, + cwd: SANDBOX_CWD, + timeoutMs: evalCommandTimeoutMs(options), }, true ); await runSandboxCommand(sandbox, label, 'validate result', { cmd: 'test', args: ['-f', `results/${pair.experiment}/${pair.eval_id}.json`], - cwd: sandbox.cwd, + cwd: SANDBOX_CWD, timeoutMs: 30_000, }); await runSandboxCommand(sandbox, label, 'pack results', { @@ -288,7 +276,7 @@ async function runPairOnce( `results/${pair.experiment}`, '.', ], - cwd: sandbox.cwd, + cwd: SANDBOX_CWD, timeoutMs: 3 * 60 * 1_000, }); await downloadResults(sandbox, pair, options.outputDir); @@ -300,94 +288,6 @@ async function runPairOnce( } } -/** - * Retries Sandbox.create() through the vCPU-provisioning rate limit. - * The SDK's own retry gives up after ~3 attempts or a >20s Retry-After, - * which isn't enough to ride out a large burst, so this layer takes over - * with a much bigger budget and honors the same Retry-After header. - */ -async function createSandbox( - label: string, - createOptions: Parameters[0] -): Promise { - return pRetry(() => Sandbox.create(createOptions), { - retries: 12, - minTimeout: 0, - shouldRetry: ({ error }) => isRetryableSandboxCreateError(error), - onFailedAttempt: async ({ error, attemptNumber, retriesLeft }) => { - const retrying = retriesLeft > 0 && isRetryableSandboxCreateError(error); - const retryAfter = getRetryAfterMs(error); - // Jitter so on top of server's `Retry-After` so concurrent pairs hitting - // the same rate limit don't all retry in lockstep. - const wait = retryAfter - ? retryAfter * (1 + Math.random() * 0.4) - : Math.min(2 ** attemptNumber * 1_000, 20_000) * - (0.8 + Math.random() * 0.4); - console.warn( - `${label} sandbox create attempt ${attemptNumber} failed${retrying ? `, retrying in ${Math.round(wait / 1_000)}s` : ', not retrying'}: ${errorMessage(error)}` - ); - if (retrying) await new Promise((resolve) => setTimeout(resolve, wait)); - }, - }); -} - -/** - * Matches the SDK's retry policy of network errors, 429s, and 5xx. - * Other 4xx responses (bad token, invalid options) can never succeed. - * https://github.com/vercel/sandbox/blob/bf2bc66003fc89cf07a1346a7ea63951747cbec6/packages/vercel-sandbox/src/api-client/with-retry.ts#L10-L17 - */ -export function isRetryableSandboxCreateError(error: unknown): boolean { - if (!(error instanceof APIError)) return true; - const { status } = error.response; - return status === 429 || status >= 500; -} - -/** - * Reads the Sandbox API's Retry-After header (seconds), converted to milliseconds. - * The SDK's own retry layer reads the same header on 429s: - * https://github.com/vercel/sandbox/blob/bf2bc66003fc89cf07a1346a7ea63951747cbec6/packages/vercel-sandbox/src/api-client/with-retry.ts#L56 - */ -function getRetryAfterMs(error: unknown): number | undefined { - if (!(error instanceof APIError)) return undefined; - const seconds = Number(error.response.headers.get('Retry-After')); - return seconds > 0 ? seconds * 1_000 : undefined; -} - -/** Runs a detached command, streams logs best-effort, and waits for its exit code. */ -async function runSandboxCommand( - sandbox: Sandbox, - label: string, - step: string, - options: SandboxCommandOptions, - logOutput = false -): Promise { - const command = await sandbox.runCommand({ - ...options, - detached: true, - }); - const result = await pRetry(() => command.wait(), { - retries: 5, - factor: 2, - minTimeout: 1_000, - maxTimeout: 10_000, - onFailedAttempt: ({ error, attemptNumber }) => { - console.warn( - `${label} ${step} wait failed (attempt ${attemptNumber}): ${errorMessage(error)}` - ); - }, - }); - if (result.exitCode === 0) { - if (logOutput) { - const output = await result.output('both').catch(() => ''); - if (output.trim()) process.stdout.write(`${label} ${output}`); - } - return; - } - - const output = await result.output('both').catch(() => ''); - throw new SandboxCommandError(step, result.exitCode, output); -} - /** Downloads and extracts one pair into the aggregate artifact directory. */ async function downloadResults( sandbox: Sandbox, @@ -444,27 +344,6 @@ export function parsePairs(value: string): EvalPair[] { return parsed.data; } -/** Returns an environment variable or a useful configuration error. */ -function requireEnv(name: string, hint: string): string { - const value = process.env[name]; - if (!value) throw new Error(`${name} is not set: ${hint}`); - return value; -} - -type VercelCredentials = ReturnType; - -/** Returns explicit credentials because the SDK does not infer all local vars. */ -function vercelCredentialsFromEnv(): - | { token: string; teamId: string; projectId: string } - | Record { - if (process.env.VERCEL_OIDC_TOKEN) return {}; - return { - token: requireEnv('VERCEL_TOKEN', 'missing Vercel token'), - teamId: requireEnv('VERCEL_TEAM_ID', 'missing Vercel team ID'), - projectId: requireEnv('VERCEL_PROJECT_ID', 'missing Vercel project ID'), - }; -} - /** Serializes configured provider keys into the repo-root `.env` file. */ function agentEnvironment(): string { const lines: string[] = []; @@ -509,30 +388,6 @@ function sandboxName(pair: EvalPair): string { return `${tagValue(pair.experiment).slice(0, 35)}--${tagValue(pair.eval_id).slice(0, 45)}--${suffix}`; } -/** Sanitizes metadata for Sandbox names and tags. */ -export function tagValue(value: string): string { - return value - .toLowerCase() - .replaceAll(/[^a-z0-9-]+/g, '-') - .slice(0, 64); -} - -const apiErrorBodySchema = z.object({ - error: z.object({ code: z.string(), message: z.string() }), -}); - -/** Converts unknown thrown values into readable diagnostics. */ -function errorMessage(error: unknown): string { - if (error instanceof APIError) { - const body = apiErrorBodySchema.safeParse(error.json); - const detail = body.success - ? `${body.data.error.code}: ${body.data.error.message}` - : error.message; - return `HTTP ${error.response.status} ${detail}`; - } - return error instanceof Error ? error.message : String(error); -} - /** Keeps retry notices readable when the final summary carries full output. */ function firstLine(value: string): string { return value.split('\n', 1)[0] ?? value; @@ -562,6 +417,9 @@ async function main(): Promise { 'concurrency' ), vcpus: positiveInteger(readFlag(rawArgs, 'vcpus') ?? '4', 'vcpus'), + // An explicit id reuses a prebuilt snapshot across runs; `--snapshot` + // builds one for this run (resolved below, after the dry-run guard). + snapshotId: readFlag(rawArgs, 'snapshot-id'), }; console.log( @@ -569,6 +427,15 @@ async function main(): Promise { ); for (const pair of options.pairs) console.log(`PLAN ${pairLabel(pair)}`); if (rawArgs.includes('--dry-run')) return; + + if (!options.snapshotId && rawArgs.includes('--snapshot')) { + options.snapshotId = await ensureSnapshot({ + root: ROOT, + repoUrl: options.repoUrl, + revision: options.revision, + vcpus: options.vcpus, + }); + } await runPairs(options); } diff --git a/apps/framework/scripts/vercel-sandbox.ts b/apps/framework/scripts/vercel-sandbox.ts new file mode 100644 index 00000000..039ddff8 --- /dev/null +++ b/apps/framework/scripts/vercel-sandbox.ts @@ -0,0 +1,245 @@ +/** + * Shared Vercel Sandbox plumbing for the eval runner and the snapshot builder: + * credentials, rate-limit-aware creation, the detached-and-polled command + * runner, and the cold-boot provisioning both a per-pair VM and the snapshot + * builder go through. Kept in its own module so neither consumer imports the + * other. + */ + +import { APIError, Sandbox } from '@vercel/sandbox'; +import pRetry from 'p-retry'; +import { z } from 'zod'; + +/** Checkout path of both git-source and snapshot-source sandboxes. */ +export const SANDBOX_CWD = '/vercel/sandbox'; + +export interface SandboxCommandOptions { + cmd: string; + args?: string[]; + cwd?: string; + env?: Record; + sudo?: boolean; + timeoutMs?: number; +} + +class SandboxCommandError extends Error { + constructor(step: string, exitCode: number, output: string) { + const detail = output.trim().slice(-4_000); + super(`${step} exited with code ${exitCode}${detail ? `\n${detail}` : ''}`); + this.name = 'SandboxCommandError'; + } +} + +/** + * Retries Sandbox.create() through the vCPU-provisioning rate limit. + * The SDK's own retry gives up after ~3 attempts or a >20s Retry-After, + * which isn't enough to ride out a large burst, so this layer takes over + * with a much bigger budget and honors the same Retry-After header. + */ +export async function createSandbox( + label: string, + createOptions: Parameters[0] +): Promise { + return pRetry(() => Sandbox.create(createOptions), { + retries: 12, + minTimeout: 0, + shouldRetry: ({ error }) => isRetryableSandboxCreateError(error), + onFailedAttempt: async ({ error, attemptNumber, retriesLeft }) => { + const retrying = retriesLeft > 0 && isRetryableSandboxCreateError(error); + const retryAfter = getRetryAfterMs(error); + // Jitter on top of the server's `Retry-After` so concurrent pairs hitting + // the same rate limit don't all retry in lockstep. + const wait = retryAfter + ? retryAfter * (1 + Math.random() * 0.4) + : Math.min(2 ** attemptNumber * 1_000, 20_000) * + (0.8 + Math.random() * 0.4); + console.warn( + `${label} sandbox create attempt ${attemptNumber} failed${retrying ? `, retrying in ${Math.round(wait / 1_000)}s` : ', not retrying'}: ${errorMessage(error)}` + ); + if (retrying) await new Promise((resolve) => setTimeout(resolve, wait)); + }, + }); +} + +/** + * Matches the SDK's retry policy of network errors, 429s, and 5xx. + * Other 4xx responses (bad token, invalid options) can never succeed. + * https://github.com/vercel/sandbox/blob/bf2bc66003fc89cf07a1346a7ea63951747cbec6/packages/vercel-sandbox/src/api-client/with-retry.ts#L10-L17 + */ +export function isRetryableSandboxCreateError(error: unknown): boolean { + if (!(error instanceof APIError)) return true; + const { status } = error.response; + return status === 429 || status >= 500; +} + +/** + * Reads the Sandbox API's Retry-After header (seconds), converted to milliseconds. + * The SDK's own retry layer reads the same header on 429s: + * https://github.com/vercel/sandbox/blob/bf2bc66003fc89cf07a1346a7ea63951747cbec6/packages/vercel-sandbox/src/api-client/with-retry.ts#L56 + */ +function getRetryAfterMs(error: unknown): number | undefined { + if (!(error instanceof APIError)) return undefined; + const seconds = Number(error.response.headers.get('Retry-After')); + return seconds > 0 ? seconds * 1_000 : undefined; +} + +/** Runs a detached command, streams logs best-effort, and waits for its exit code. */ +export async function runSandboxCommand( + sandbox: Sandbox, + label: string, + step: string, + options: SandboxCommandOptions, + logOutput = false +): Promise { + const command = await sandbox.runCommand({ + ...options, + detached: true, + }); + const result = await pRetry(() => command.wait(), { + retries: 5, + factor: 2, + minTimeout: 1_000, + maxTimeout: 10_000, + onFailedAttempt: ({ error, attemptNumber }) => { + console.warn( + `${label} ${step} wait failed (attempt ${attemptNumber}): ${errorMessage(error)}` + ); + }, + }); + if (result.exitCode === 0) { + if (logOutput) { + const output = await result.output('both').catch(() => ''); + if (output.trim()) process.stdout.write(`${label} ${output}`); + } + return; + } + + const output = await result.output('both').catch(() => ''); + throw new SandboxCommandError(step, result.exitCode, output); +} + +/** + * Starts dockerd inside the VM and waits for the socket. On cold boots Docker + * must be installed first; on warm boots the snapshot already carries it, so + * only the daemon (which isn't part of a filesystem snapshot) is restarted. + */ +export async function startDocker( + sandbox: Sandbox, + label: string, + { install }: { install: boolean } +): Promise { + if (install) { + await runSandboxCommand(sandbox, label, 'install Docker', { + cmd: 'dnf', + args: ['install', '-y', '-q', 'docker'], + sudo: true, + timeoutMs: 3 * 60 * 1_000, + }); + } + await sandbox.runCommand({ cmd: 'dockerd', sudo: true, detached: true }); + await runSandboxCommand(sandbox, label, 'start Docker', { + cmd: 'bash', + args: [ + '-c', + 'for i in $(seq 60); do docker info >/dev/null 2>&1 && chmod 666 /var/run/docker.sock && exit 0; sleep 1; done; echo "dockerd not ready" >&2; exit 1', + ], + sudo: true, + timeoutMs: 90_000, + }); +} + +/** Installs the workspace's pinned dependencies (a cache hit on warm boots). */ +export async function installDependencies( + sandbox: Sandbox, + label: string +): Promise { + await runSandboxCommand(sandbox, label, 'install dependencies', { + cmd: 'pnpm', + args: ['install', '--frozen-lockfile'], + cwd: SANDBOX_CWD, + timeoutMs: 6 * 60 * 1_000, + }); +} + +/** + * Full cold-boot provisioning — what both a per-pair VM (cold branch) and the + * snapshot builder run before they can `pnpm eval`: init submodules, install + + * start Docker, install the pinned pnpm, and install dependencies. + */ +export async function coldProvision( + sandbox: Sandbox, + label: string +): Promise { + await runSandboxCommand(sandbox, label, 'initialize submodules', { + cmd: 'git', + args: [ + '-c', + 'url.https://github.com/.insteadOf=git@github.com:', + 'submodule', + 'update', + '--init', + '--recursive', + ], + cwd: SANDBOX_CWD, + timeoutMs: 2 * 60 * 1_000, + }); + await startDocker(sandbox, label, { install: true }); + // Installs the pnpm version pinned by packageManager in the checked-out root + // package.json so the two can't drift. + await runSandboxCommand(sandbox, label, 'install pnpm', { + cmd: 'bash', + args: [ + '-c', + `npm install --global "$(node -p 'require("./package.json").packageManager')"`, + ], + cwd: SANDBOX_CWD, + sudo: true, + timeoutMs: 2 * 60 * 1_000, + }); + await installDependencies(sandbox, label); +} + +/** Returns an environment variable or a useful configuration error. */ +function requireEnv(name: string, hint: string): string { + const value = process.env[name]; + if (!value) throw new Error(`${name} is not set: ${hint}`); + return value; +} + +export type VercelCredentials = ReturnType; + +/** Returns explicit credentials because the SDK does not infer all local vars. */ +export function vercelCredentialsFromEnv(): + | { token: string; teamId: string; projectId: string } + | Record { + if (process.env.VERCEL_OIDC_TOKEN) return {}; + return { + token: requireEnv('VERCEL_TOKEN', 'missing Vercel token'), + teamId: requireEnv('VERCEL_TEAM_ID', 'missing Vercel team ID'), + projectId: requireEnv('VERCEL_PROJECT_ID', 'missing Vercel project ID'), + }; +} + +const apiErrorBodySchema = z.object({ + error: z.object({ code: z.string(), message: z.string() }), +}); + +/** Converts unknown thrown values into readable diagnostics. */ +export function errorMessage(error: unknown): string { + if (error instanceof APIError) { + const body = apiErrorBodySchema.safeParse(error.json); + const detail = body.success + ? `${body.data.error.code}: ${body.data.error.message}` + : error.message; + return `HTTP ${error.response.status} ${detail}`; + } + return error instanceof Error ? error.message : String(error); +} + +/** Sanitizes metadata for Sandbox names and tags. */ +export function tagValue(value: string): string { + return value + .toLowerCase() + .replaceAll(/[^a-z0-9-]+/g, '-') + .slice(0, 64); +} diff --git a/apps/framework/scripts/vercel-snapshot.ts b/apps/framework/scripts/vercel-snapshot.ts new file mode 100644 index 00000000..fda61d41 --- /dev/null +++ b/apps/framework/scripts/vercel-snapshot.ts @@ -0,0 +1,194 @@ +/** + * Warm-boot snapshots: pre-bake everything a per-pair VM spends its cold setup + * on — docker, pnpm + node_modules, the agent sandbox base image, and the + * pulled Supabase stack images — into a Vercel Sandbox snapshot, so per-pair + * VMs boot from it and only restart dockerd, fetch the target revision, and + * reconcile any dependency drift. + * + * The build is a one-shot: `ensureSnapshot` provisions a builder VM, runs the + * same cold-boot steps a pair does (`coldProvision`) plus the two image bakes, + * tears the prewarmed stack fully down so only *images* survive into the + * snapshot (a leftover container/volume/network would change the environment a + * warm-booted agent sees), and snapshots the filesystem. + * + * `@vercel/sandbox@3`'s `listSnapshots` can't filter by name, so cross-run + * reuse is done by passing the returned id back via `--snapshot-id` rather than + * rediscovering it here. + */ + +import { createHash } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { + SKILLS_CLI_VERSION, + SUPABASE_CLI_VERSION, +} from '@supabase-evals/sandbox'; +import { + SANDBOX_CWD, + coldProvision, + createSandbox, + errorMessage, + runSandboxCommand, + vercelCredentialsFromEnv, +} from './vercel-sandbox.js'; + +/** Bump to invalidate the key when the builder steps themselves change. */ +const BUILDER_VERSION = '1'; +/** One-time builder budget; the Supabase image pulls dominate it. */ +const BUILDER_TIMEOUT_MS = 30 * 60 * 1_000; +const LABEL = '[snapshot]'; + +export interface EnsureSnapshotOptions { + root: string; + repoUrl: string; + revision: string; + vcpus: number; +} + +/** + * Fingerprint of everything the snapshot's contents depend on: the dependency + * tree, the agent image definition + its pinned versions, and the builder + * itself. Used only to name the builder VM (a per-pair `pnpm install` + * reconciles any drift between this and the run's revision). + */ +export function computeSnapshotKey(root: string): string { + return createHash('sha256') + .update(`builder:${BUILDER_VERSION}\n`) + .update(`supabase:${SUPABASE_CLI_VERSION}\n`) + .update(`skills:${SKILLS_CLI_VERSION}\n`) + .update(readFileSync(join(root, 'packages/sandbox/Dockerfile'))) + .update(readFileSync(join(root, 'pnpm-lock.yaml'))) + .digest('hex') + .slice(0, 12); +} + +/** + * Build a warm-boot snapshot and return its id, or undefined on any failure — + * the snapshot is an optimization, never a requirement, so the caller falls + * back to cold git-source boots. + */ +export async function ensureSnapshot( + options: EnsureSnapshotOptions +): Promise { + try { + // The key names the builder for traceability; a random suffix keeps repeat + // builds from colliding on the (still-lingering) previous builder's name — + // v3 can't resolve snapshots by name anyway, so uniqueness costs nothing. + const suffix = Math.random().toString(36).slice(2, 8); + const name = `evals-snap-${computeSnapshotKey(options.root)}-${suffix}`; + console.log(`${LABEL} building warm-boot snapshot ${name} (~3-10m, one-time)`); + const start = Date.now(); + const id = await buildSnapshot(name, options); + const minutes = Math.round((Date.now() - start) / 60_000); + console.log(`${LABEL} built ${id} in ${minutes}m — reuse with --snapshot-id`); + return id; + } catch (error) { + console.warn( + `${LABEL} unavailable, falling back to cold boots: ${errorMessage(error)}` + ); + return undefined; + } +} + +async function buildSnapshot( + name: string, + options: EnsureSnapshotOptions +): Promise { + const sandbox = await createSandbox(LABEL, { + ...vercelCredentialsFromEnv(), + name, + runtime: 'node24', + source: { + type: 'git', + url: options.repoUrl, + revision: options.revision, + depth: 1, + }, + resources: { vcpus: options.vcpus }, + timeout: BUILDER_TIMEOUT_MS, + // Bound this builder to a single retained snapshot (belt-and-braces against + // an auto-snapshot-on-stop landing alongside our explicit one). + keepLastSnapshots: { count: 1 }, + tags: { + runner: 'supabase-evals', + run: process.env.GITHUB_RUN_ID ?? 'local', + purpose: 'snapshot-builder', + }, + }); + console.log(`${LABEL} builder sandbox ${sandbox.name} created`); + try { + // The exact cold-boot provisioning a per-pair VM does, so a warm boot only + // adds the target-revision checkout on top of an identical base. + await coldProvision(sandbox, LABEL); + + // --- The two bakes that make warm boots pay off. --- + // The exact tag + build args ensureSupabaseSandboxImage uses, so its + // `docker image inspect` is a cache hit at eval time. + await runSandboxCommand(sandbox, LABEL, 'build agent image', { + cmd: 'bash', + args: [ + '-c', + `docker build --build-arg SKILLS_CLI_VERSION=${SKILLS_CLI_VERSION} ` + + `--tag supabase-evals-sandbox:base-skills-${SKILLS_CLI_VERSION} ` + + `- < packages/sandbox/Dockerfile`, + ], + cwd: SANDBOX_CWD, + timeoutMs: 10 * 60 * 1_000, + }); + // Pre-pull the Supabase stack: the pinned CLI on the VM host runs a + // throwaway `supabase start` (pulls every service image into the daemon, + // which is snapshotted). The VM is Amazon Linux (rpm); the agent container + // is Debian (.deb) — same pinned version, so the same image tags get + // pulled and the agent's own `supabase start` is a cache hit. + await runSandboxCommand(sandbox, LABEL, 'install supabase CLI', { + cmd: 'bash', + args: [ + '-c', + `ARCH="$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/')" && ` + + `curl -fsSL "https://github.com/supabase/cli/releases/download/v${SUPABASE_CLI_VERSION}/supabase_${SUPABASE_CLI_VERSION}_linux_$ARCH.rpm" -o /tmp/supabase.rpm && ` + + `rpm -i /tmp/supabase.rpm && rm /tmp/supabase.rpm`, + ], + sudo: true, + timeoutMs: 2 * 60 * 1_000, + }); + await runSandboxCommand(sandbox, LABEL, 'prewarm supabase images', { + cmd: 'bash', + args: [ + '-c', + 'mkdir -p /tmp/prewarm && cd /tmp/prewarm && supabase init && supabase start && supabase stop --no-backup && cd / && rm -rf /tmp/prewarm', + ], + timeoutMs: 15 * 60 * 1_000, + }); + + // Fidelity guard: only *images* may survive into the snapshot. Force-remove + // any container/volume/network the prewarm left so a warm-booted agent gets + // the same clean daemon a cold boot would (no port clashes, stale state, or + // pre-existing project). Images are kept — that's the whole point. + await runSandboxCommand(sandbox, LABEL, 'reset docker state', { + cmd: 'bash', + args: [ + '-c', + 'docker ps -aq | xargs -r docker rm -f\n' + + 'docker volume ls -q | xargs -r docker volume rm -f\n' + + 'docker network prune -f\n' + + 'remaining="$(docker ps -aq)"; if [ -n "$remaining" ]; then echo "containers survived prewarm: $remaining" >&2; exit 1; fi', + ], + timeoutMs: 2 * 60 * 1_000, + }); + // Nothing run-specific may live in the snapshot. + await runSandboxCommand(sandbox, LABEL, 'scrub', { + cmd: 'bash', + args: ['-c', 'rm -f .env /tmp/eval-results.tgz /tmp/step-*'], + cwd: SANDBOX_CWD, + timeoutMs: 30_000, + }); + + console.log(`${LABEL} snapshotting (stops the builder)…`); + const snapshot = await sandbox.snapshot(); + return snapshot.snapshotId; + } catch (error) { + // snapshot() stops the sandbox on success; on failure stop it ourselves. + await sandbox.stop().catch(() => undefined); + throw error; + } +} diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index 006be91c..3556ddcf 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -31,7 +31,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user d7fb398a-0a59-40f5-aaa8-bc0b26722470, signUp returned {\"userId\":\"d7fb398a-0a59-40f5-aaa8-bc0b26722470\"}" + "notes": "db user 8daedd93-5c0a-4825-975e-aad69021ed0c, signUp returned {\"userId\":\"8daedd93-5c0a-4825-975e-aad69021ed0c\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -46,7 +46,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"d7fb398a-0a59-40f5-aaa8-bc0b26722470\"}" + "notes": "{\"userId\":\"8daedd93-5c0a-4825-975e-aad69021ed0c\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -75,37 +75,15 @@ }, "docs": { "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"signUp email password options data user metadata javascript\", limit: 4) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", - "title": "Customizing Emails by Language" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signup" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" - } - ], - "resultChars": 22309 - }, { "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md | head -60", + "query": "curl -s https://supabase.com/changelog.md | head -80", "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 3161 + "resultChars": 4557 } ] }, @@ -183,23 +161,13 @@ "calls": [ { "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md 2>&1 | grep -iE 'breaking|rls|policy|grant' | head -30", + "query": "curl -s https://supabase.com/changelog.md | head -60", "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 5391 - }, - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog/45329-breaking-change-tables-not-exposed-to-data-and-graphql-api-automatically.md 2>&1 | head -60", - "pages": [ - { - "url": "https://supabase.com/changelog/45329-breaking-change-tables-not-exposed-to-data-and-graphql-api-automatically.md" - } - ], - "resultChars": 3976 + "resultChars": 3161 } ] }, @@ -258,18 +226,7 @@ ] }, "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/docs/guides/local-development/declarative-database-schemas.md | head -120", - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas.md" - } - ], - "resultChars": 4170 - } - ] + "calls": [] }, "prompt": "Add a description text column to the `products` table in my local Supabase stack", "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", @@ -314,7 +271,7 @@ { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 83) from the queue" + "notes": "function removed the seeded message (id 40) from the queue" } ], "skills": { @@ -331,68 +288,88 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"pg_cron schedule job invoke edge function pgmq queue\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"pg_cron schedule job queues pgmq send message\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" }, { "url": "https://supabase.com/docs/guides/database/extensions/pg_net", "title": "pg_net: Async Networking" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" + "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", + "title": "pg_cron debugging guide" }, { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", - "title": "pg_cron debugging guide" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" }, { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" + "url": "https://supabase.com/docs/guides/cron/install", + "title": "Install" } ], - "resultChars": 62989 + "resultChars": 83097 }, { - "source": "web_fetch", - "query": "How do you create a cron job that runs every minute using SQL (cron.schedule)? Show the exact SQL syntax, including how to name a job, how to run SQL vs invoke an edge function, and how to unschedule/replace an existing job. Include any notes about job names and idempotency.", - "hasContent": true, + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md 2>&1 | head -60", "pages": [ { - "url": "https://supabase.com/docs/guides/cron/quickstart.md" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 1310 + "resultChars": 3161 }, { - "source": "web_fetch", - "query": "How do you create a pgmq queue and send/read/delete messages? Show exact SQL function signatures for pgmq.create, pgmq.send, pgmq.read, pgmq.delete, pgmq.pop. Also note anything about queue types (basic/unlogged/partitioned) and the pgmq_public wrappers for the Data API.", - "hasContent": true, + "source": "shell_fetch", + "query": "curl -s https://supabase.com/docs/guides/queues/quickstart.md | head -120", "pages": [ { "url": "https://supabase.com/docs/guides/queues/quickstart.md" } ], - "resultChars": 1391 + "resultChars": 6263 }, { - "source": "web_fetch", - "query": "List any entries tagged breaking-change or otherwise notable that relate to: Queues / pgmq, Cron / pg_cron, or Edge Functions (deployment, config.toml verify_jwt, serving). Include dates and links.", - "hasContent": true, + "source": "shell_fetch", + "query": "curl -s https://supabase.com/docs/guides/cron.md | head -100", "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/cron.md" + } + ], + "resultChars": 1404 + }, + { + "source": "shell_fetch", + "query": "curl -s https://supabase.com/docs/guides/queues/api.md | head -80; echo \"=== PGMQ EXT DOC ===\"; curl -s https://supabase.com/docs/guides/queues/pgmq.md | grep -iA22 'read.*message\\|## read' | head -60", + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues/api.md" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq.md" + } + ], + "resultChars": 4291 + }, + { + "source": "shell_fetch", + "query": "curl -s https://supabase.com/docs/guides/functions/quickstart.md | head -70", + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/quickstart.md" } ], - "resultChars": 3048 + "resultChars": 3022 } ] }, @@ -460,36 +437,7 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"secret key service role server-side supabase-js createClient API keys\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/performing-administration-tasks-on-the-server-side-with-the-servicerole-secret-BYM4Fa", - "title": "Performing administration tasks on the server side with a secret key" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/quickstarts/sveltekit", - "title": "Use Supabase with SvelteKit" - } - ], - "resultChars": 94242 - } - ] + "calls": [] }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", @@ -551,12 +499,22 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "ls -a app/ && cat skills-lock.json && curl -s https://supabase.com/changelog.md | head -60", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 3608 + } + ] }, "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", @@ -616,7 +574,40 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"restore pg_dump backup pg_restore migrate existing postgres database to Supabase\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", + "title": "Migrate from Vercel Postgres to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/neon", + "title": "Migrate from Neon to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/heroku", + "title": "Migrate from Heroku to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", + "title": "Migrate from Postgres to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/dashboard-restore", + "title": "Restore Dashboard backup" + } + ], + "resultChars": 55834 + } + ] }, "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", @@ -666,12 +657,12 @@ { "name": "user A cannot force-read user B note", "passed": true, - "notes": "status=403" + "notes": "status=200" }, { "name": "user B cannot force-read user A note", "passed": true, - "notes": "status=403" + "notes": "status=200" } ], "skills": { @@ -684,70 +675,7 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Function auth user Authorization header RLS createClient\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/functions/http-methods", - "title": "Routing" - } - ], - "resultChars": 73347 - }, - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/docs/guides/functions/auth.md | head -120", - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth.md" - } - ], - "resultChars": 5299 - }, - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/docs/guides/functions/auth-headers.md", - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-headers.md" - } - ], - "resultChars": 3330 - }, - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/docs/guides/functions/secrets.md | head -60", - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets.md" - } - ], - "resultChars": 2599 - } - ] + "calls": [] }, "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", @@ -778,7 +706,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": true, + "passed": false, "checks": [ { "name": "seed rows present", @@ -788,42 +716,42 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"error\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"07c687de-fcad-4acd-9271-a501b42d02be\",\"metric\":\"steps_a_msj188pa\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"a987e414-ddf1-4b93-be2c-df4aa483c268\",\"metric\":\"steps_a_mssz63em\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: [{\"user_id\":\"07c687de-fcad-4acd-9271-a501b42d02be\",\"metric\":\"steps_a_msj188pa\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"a987e414-ddf1-4b93-be2c-df4aa483c268\",\"metric\":\"steps_a_mssz63em\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"831c5088-6bc6-4c0e-8c1f-3fe53e8a0852\",\"metric\":\"steps_b_msj188pa\",\"value\":222}]" + "notes": "status 200: [{\"user_id\":\"a28b39c4-f7e0-44e6-a5a6-a069b6c5e826\",\"metric\":\"steps_b_mssz63em\",\"value\":222}]" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"error\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"error\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "status 401: {\"error\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { "name": "implementation uses @supabase/server", - "passed": true, - "notes": "imports @supabase/server / withSupabase" + "passed": false, + "notes": "hand-rolled (raw supabase-js or other) — this eval requires @supabase/server" } ], "skills": { @@ -837,87 +765,103 @@ }, "docs": { "calls": [ - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md 2>/dev/null | head -60 || echo \"changelog fetch failed\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 3161 - }, - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog/45702-developer-update-may-2026.md 2>/dev/null | head -100", - "pages": [ - { - "url": "https://supabase.com/changelog/45702-developer-update-may-2026.md" - } - ], - "resultChars": 6160 - }, { "source": "search_docs", - "query": "{ searchDocs(query: \"@supabase/server SDK edge functions auth service role\", limit: 12) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"edge function verify_jwt service role key apikey header authentication\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" }, { "url": "https://supabase.com/docs/guides/functions/auth", "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/auth/third-party/auth0", - "title": "Auth0" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/firebase-auth", - "title": "Firebase Auth" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" }, { - "url": "https://supabase.com/docs/guides/auth/third-party/aws-cognito", - "title": "Amazon Cognito (Amplify)" + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" }, { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/guides/auth/third-party/workos", - "title": "WorkOS" - }, + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + } + ], + "resultChars": 118701 + }, + { + "source": "web_fetch", + "query": "List any entries about Edge Functions authentication, verify_jwt, new API keys (publishable/secret keys, sb_publishable_, sb_secret_), @supabase/server package, or breaking changes affecting edge function auth or API key env vars. Include dates and links.", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", - "title": "Custom Roles" - }, + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 1189 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"@supabase/server SDK edge functions\", limit: 3) { nodes { title href } } }", + "hasContent": false, + "pages": [ { - "url": "https://supabase.com/docs/reference/cli/supabase-functions", - "title": "Manage Supabase Edge functions" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/auth/architecture", - "title": "Auth architecture" + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" }, { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + } + ], + "resultChars": 407 + }, + { + "source": "web_fetch", + "query": "Full content: how to handle auth in edge functions, verify_jwt, service role key, creating supabase client with user token, getUser vs getClaims, apikey header. Include code samples verbatim.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth.md" + } + ], + "resultChars": 1982 + }, + { + "source": "shell_fetch", + "query": "cd /tmp && curl -sS https://supabase.com/docs/guides/functions/auth-headers.md | head -200", + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth-headers.md" } ], - "resultChars": 75829 + "resultChars": 3375 } ] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5/build-functions-005-dual-auth-user-secret.json" }, { @@ -958,7 +902,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019fdc90-5069-75c6-93dc-d42f28e61c92/receipt-alpha.pdf, 019fdc90-5069-75c6-93dc-d42f28e61c92/receipt-beta.pdf" + "notes": "saw: 01a00060-bcce-767c-b73e-274bb5d5d4e9/receipt-alpha.pdf, 01a00060-bcce-767c-b73e-274bb5d5d4e9/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -979,7 +923,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Meets all required criteria: private user-files bucket, owner-scoped SELECT and INSERT RLS policies on storage.objects for authenticated users with RLS kept enabled, and supabase-js createSignedUrl code with expiry for temporary sharing. No disqualifying public bucket, permissive policies, public URL sharing, or client service-role usage." + "judgeNotes": "Configures a private user-files bucket, owner-scoped SELECT and INSERT RLS policies on storage.objects with RLS kept enabled, and supabase-js createSignedUrl code with expiry for temporary sharing. No disqualifying public bucket, permissive policy, getPublicUrl, or client-side service role usage." } ], "skills": { @@ -995,7 +939,7 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"storage RLS policies private bucket user folder owner uid\", limit: 6) { nodes { title href content } } }", + "query": "{searchDocs(query:\"storage RLS policies private bucket user folder owner uid\", limit:5){nodes{title href content}}}", "hasContent": true, "pages": [ { @@ -1017,30 +961,29 @@ { "url": "https://supabase.com/docs/guides/local-development/database-migrations", "title": "Database migrations" - }, - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" } ], - "resultChars": 36724 + "resultChars": 32975 }, { - "source": "web_fetch", - "query": "List any entries tagged breaking-change, and any entries related to Storage buckets, storage RLS policies, signed URLs, or the storage.objects schema (e.g. owner/owner_id columns, new bucket types). Include dates and links.", - "hasContent": true, + "source": "shell_fetch", + "query": "curl -s --max-time 25 https://supabase.com/changelog.md | head -100", "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 8117 + "resultChars": 5516 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"createSignedUrl expires in temporary link download share\", limit: 4) { nodes { title href content } } }", + "query": "{searchDocs(query:\"createSignedUrl expiresIn signed URL download storage javascript\", limit:4){nodes{title href content}}}", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", + "title": "Storage Image Transformations" + }, { "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" }, @@ -1049,13 +992,9 @@ }, { "url": "https://supabase.com/docs/reference/dart/file-buckets-createsignedurl" - }, - { - "url": "https://supabase.com/docs/reference/csharp/storage-from-createsignedurl", - "title": "From().CreateSignedUrl()" } ], - "resultChars": 3099 + "resultChars": 22012 } ] }, @@ -1089,17 +1028,17 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "3 file(s): supabase/tests/002_tenant_isolation_writes.test.sql, supabase/tests/001_tenant_isolation_select.test.sql, supabase/tests/003_tenant_isolation_baseline.test.sql" + "notes": "1 file(s): supabase/tests/tenant_isolation.test.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "7 passed, 5 failed" + "notes": "11 passed, 11 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as having the broken tenant isolation policy, explicitly noting authenticated users in any org can read every post because the policy is missing `m.org_id = posts.org_id`. It grounds this in pgTAP results, distinguishes `notes` as passing read isolation, and treats test results as authoritative rather than trusting migration comments." + "judgeNotes": "The agent correctly identifies `posts` as having the broken tenant isolation policy, explains that authenticated members of any org can read all posts due to the missing `m.org_id = posts.org_id` predicate, and grounds this in pgTAP failures and direct verification. It does not blame `notes`; it explicitly says `notes` is correctly isolated." } ], "skills": { @@ -1108,12 +1047,40 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"pgTAP database testing RLS policies supabase test db\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/testing/overview", + "title": "Testing Overview" + }, + { + "url": "https://supabase.com/docs/guides/database/testing", + "title": "Testing Your Database" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", + "title": "Testing and linting" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" + } + ], + "resultChars": 57796 + } + ] }, "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", @@ -1188,36 +1155,48 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "{ searchDocs(query: \"semantic search pgvector gte-small edge function embeddings match_document_sections RLS\", limit: 6) { nodes { title href content } } }", + "source": "web_fetch", + "query": "Show the full recommended SQL for a match_document_sections style vector search function, including the embedding column definition, HNSW index, RLS setup, and how the function handles security invoker. Include exact SQL verbatim.", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" - }, - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - }, - { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, + "url": "https://supabase.com/docs/guides/ai/semantic-search.md" + } + ], + "resultChars": 1413 + }, + { + "source": "web_fetch", + "query": "Show verbatim SQL for creating a vector column, the vector extension schema recommendation, HNSW index creation with operator class for normalized embeddings (inner product vs cosine), and the recommended match function signature.", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" - }, + "url": "https://supabase.com/docs/guides/ai/vector-columns.md" + } + ], + "resultChars": 1320 + }, + { + "source": "web_fetch", + "query": "Show verbatim all SQL: documents/document_sections tables, RLS policies for owner-only access, the match_document_sections function signature and body, and any index. Include the exact security invoker / search_path settings used.", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions.md" + } + ], + "resultChars": 1271 + }, + { + "source": "web_fetch", + "query": "Show verbatim SQL for creating an HNSW index, including the operator classes available (vector_cosine_ops, vector_ip_ops, vector_l2_ops) and which to use for normalized embeddings with inner product.", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/ai/vector-columns", - "title": "Vector columns" + "url": "https://supabase.com/docs/guides/ai/vector-indexes/hnsw-indexes.md" } ], - "resultChars": 84107 + "resultChars": 949 } ] }, @@ -1253,12 +1232,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "judgeNotes": "Meets requirements: HTTPS Supabase Metrics API scrape for evalshostedprojectxy.supabase.co at /customer/v1/privileged/metrics, uses HTTP Basic Auth with password_file, preserves existing app scrape, and docker-compose mounts the secrets directory matching the password_file path." + "judgeNotes": "Meets requirements: existing app scrape preserved; Supabase scrape uses HTTPS, correct metrics path, Basic Auth with password_file, hosted .supabase.co target, and docker-compose mounts the password file path via the secrets directory volume." }, { "name": "documented live deployment and verification steps", "passed": true, - "judgeNotes": "README includes Secret API key creation, matching password_file setup, reload/Compose rollout steps, and concrete verification via curl, Prometheus targets/query API, and Grafana dashboard." + "judgeNotes": "README includes creating an sb_secret API key, writing it to the matching mounted secret file, bringing/reloading the stack as needed, and concrete verification via Prometheus target health/API plus direct curl credential check." } ], "skills": { @@ -1274,42 +1253,51 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"Prometheus metrics endpoint project observability scrape\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Prometheus metrics endpoint project metrics scrape\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" + "url": "https://supabase.com/docs/reference/api/v1-scrape-project-metrics", + "title": "Scrape a project's metrics" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, { "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", "title": "Vendor-agnostic Metrics API setup" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", + "title": "Metrics API" }, { "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", "title": "Metrics API with Grafana Cloud" - }, + } + ], + "resultChars": 20571 + }, + { + "source": "shell_fetch", + "query": "curl -s --max-time 25 https://supabase.com/changelog.md | head -60", + "pages": [ { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 29057 + "resultChars": 3161 }, { - "source": "web_fetch", - "query": "What is the exact metrics endpoint URL format, what authentication does it use (username/password), and what is the recommended Prometheus scrape config and scrape interval? Include any notes about rate limits or which key to use.", - "hasContent": true, + "source": "shell_fetch", + "query": "curl -s --max-time 25 https://supabase.com/changelog.md | grep -i -n -E 'metric|observab|prometheus|grafana|api key|api-key' | head -30", "pages": [ { - "url": "https://supabase.com/docs/guides/telemetry/metrics.md" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 1186 + "resultChars": 3665 } ] }, @@ -1371,9 +1359,13 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Functions secrets environment variables managing\", limit: 5) { nodes { title href content } } }", + "query": "{searchDocs(query:\"edge function secrets environment variables deploy\",limit:5){nodes{title href content}}}", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", + "title": "Inspecting edge function environment variables" + }, { "url": "https://supabase.com/docs/guides/functions/secrets", "title": "Environment Variables" @@ -1383,62 +1375,35 @@ "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", + "title": "Transcription Telegram Bot" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", + "title": "Streaming Speech with ElevenLabs" } ], - "resultChars": 71163 + "resultChars": 41423 }, { - "source": "web_fetch", - "query": "List any breaking changes or notable changes related to Edge Functions, function deployment, function secrets/environment variables, or the CLI's secrets/functions commands.", - "hasContent": true, + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md | head -60", "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 1947 - }, - { - "source": "web_fetch", - "query": "How do you set and access environment variables/secrets in Supabase Edge Functions? Include the exact CLI commands for setting production secrets from a .env file, local .env handling, and reserved secret names. Quote commands verbatim.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets.md" - } - ], - "resultChars": 1274 - }, - { - "source": "web_fetch", - "query": "Show the current recommended boilerplate for a Supabase Edge Function index.ts (Deno.serve vs std serve import), the deploy command, and any config.toml settings for functions like verify_jwt. Quote code verbatim.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/quickstart.md" - } - ], - "resultChars": 879 + "resultChars": 3161 }, { "source": "shell_fetch", - "query": "curl -s https://supabase.com/docs/guides/functions/quickstart.md | head -120", + "query": "curl -s https://supabase.com/changelog.md | grep -iE \"edge function|secret|functions deploy\" | head -30", "pages": [ { - "url": "https://supabase.com/docs/guides/functions/quickstart.md" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 5029 + "resultChars": 5478 } ] }, @@ -1500,25 +1465,14 @@ "calls": [ { "source": "web_fetch", - "query": "Extract the full self-hosting with Docker guide verbatim-ish: the exact steps to get the docker setup, all the secrets/env vars that must be changed before going to production (JWT secret, anon/service keys, postgres password, dashboard user/pass, secret_key_base, vault enc key, SITE_URL, API_EXTERNAL_URL, SUPABASE_PUBLIC_URL, pooler tenant id, etc.), securing services, and any notes about restarting/regenerating keys.", + "query": "Extract the full self-hosting-with-Docker instructions verbatim in detail: the steps to get the docker files, every secret/env var that must be changed before going to production (POSTGRES_PASSWORD, JWT_SECRET, ANON_KEY, SERVICE_ROLE_KEY, DASHBOARD credentials, SECRET_KEY_BASE, VAULT_ENC_KEY, logflare tokens, pooler tenant id etc), how API keys are generated/signed, and all \"securing your services\" guidance. Also note anything about publishable/secret keys vs anon/service_role, and any notes about the .env file, restarting, and exposed ports.", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/self-hosting/docker.md" } ], - "resultChars": 3113 - }, - { - "source": "web_fetch", - "query": "List any entries tagged breaking-change or otherwise relevant to self-hosting, Docker, JWT signing keys / API key format (publishable/secret vs anon/service_role), Supavisor/pooler, or Studio, from 2025 and 2026. Include dates and links.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 3915 + "resultChars": 6486 } ] }, @@ -1578,7 +1532,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "The answer identifies the original soft-delete-only bug and lack of session/refresh-token revocation, implements real auth/session revocation via banning and deleting sessions plus RLS checks, and explains the remaining stateless JWT caveat consistently: Data API notes/profiles are closed by RLS, but purely local JWT validation remains valid until exp with mitigations. It also correctly distinguishes publishable frontend keys with RLS from server-only secret/service_role keys that bypass RLS and must not ship to clients." + "judgeNotes": "Meets rubric: diagnoses soft-delete-only flow with no auth/session revocation; implements real session/refresh-token revocation and RLS session/user checks; explains JWT access tokens remain valid for local validation until expiry while Data API path is closed by its RLS fix; correctly distinguishes publishable frontend key vs secret server-only RLS-bypassing key." } ], "skills": { @@ -1594,24 +1548,24 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"publishable key secret key new API keys anon service_role RLS\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"publishable key secret key API keys migration anon service_role\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", - "title": "Creating a Supabase client for SSR" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { "url": "https://supabase.com/docs/guides/auth/signing-keys", "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", + "title": "Supabase for Platforms" }, { "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", @@ -1622,38 +1576,184 @@ "title": "Build a Social Auth App with Expo React Native" } ], - "resultChars": 190908 + "resultChars": 166856 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"delete user does not invalidate access token revoke sessions sign out\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"delete user account revoke sessions access token still valid\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, { "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", "title": "OAuth 2.1 Flows" }, { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-signout" + "url": "https://supabase.com/docs/guides/platform/temporary-access", + "title": "Temporary access" }, { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" + "url": "https://supabase.com/docs/guides/platform/mfa/org-mfa-enforcement", + "title": "Enforce MFA on Organization" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V", - "title": "Should I set a shorter Max-Age parameter on the cookies?" + "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", + "title": "Multiple SSO Providers" + }, + { + "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration", + "title": "Build a Supabase Integration" + } + ], + "resultChars": 82966 + }, + { + "source": "web_fetch", + "query": "Quote verbatim: the full method signature/parameters table for deleteUser, including the shouldSoftDelete parameter and its description, any notes/warnings about sessions, refresh tokens, or access tokens. Include all example code and all prose.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser.md" + } + ], + "resultChars": 215 + }, + { + "source": "web_fetch", + "query": "Quote verbatim: full signature and parameters for signOut, especially the `scope` option and ALL allowed values with their exact descriptions (global, local, others). Include all prose and notes.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-signout.md" + } + ], + "resultChars": 215 + }, + { + "source": "web_fetch", + "query": "Quote verbatim: does this page exist? Full signature, parameters (jwt, scope), allowed scope values, all prose, notes, and examples.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-admin-signout.md" + } + ], + "resultChars": 215 + }, + { + "source": "web_fetch", + "query": "Quote verbatim everything about: how sessions terminate/are revoked, refresh tokens, access token JWT expiry, whether access tokens can be revoked before exp, signOut scopes, banning users / banned_until, and admin session revocation. Include exact quotes.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/sessions.md" + } + ], + "resultChars": 1475 + }, + { + "source": "web_fetch", + "query": "Quote verbatim everything about: deleting users, banning users / banned_until, ban_duration, foreign keys referencing auth.users.id, ON DELETE CASCADE, whether to modify auth schema directly via SQL, and any warnings.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/users.md" + } + ], + "resultChars": 680 + }, + { + "source": "web_fetch", + "query": "Quote verbatim: the recommended pattern for public tables referencing auth.users.id, the exact SQL example (profiles table, references auth.users on delete cascade), any warnings about modifying the auth schema directly or deleting from auth.users via SQL.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/managing-user-data.md" + } + ], + "resultChars": 1258 + }, + { + "source": "web_fetch", + "query": "Quote verbatim the deleteUser signature, all parameters including shouldSoftDelete and its description, and any notes about sessions/tokens.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" + } + ], + "resultChars": 595 + }, + { + "source": "web_fetch", + "query": "Quote verbatim the signOut signature, the scope parameter and ALL its allowed values with exact descriptions, and any notes.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-signout" + } + ], + "resultChars": 690 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"ban user banned_until access token still valid delete user sessions revoked\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/temporary-access", + "title": "Temporary access" + }, + { + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" + }, + { + "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", + "title": "Multiple SSO Providers" + } + ], + "resultChars": 38123 + }, + { + "source": "web_fetch", + "query": "Quote verbatim anything about banning users, ban duration, banned_until, and whether a ban affects existing access tokens or only new sign-ins/refresh.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/general-configuration.md" } ], - "resultChars": 66489 + "resultChars": 465 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"delete user auth admin deleteUser cascade foreign key profiles on delete cascade\", limit: 2) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" + } + ], + "resultChars": 7291 + }, + { + "source": "web_fetch", + "query": "Does this page document admin.signOut? Quote the signature, parameters (jwt, scope), and any notes verbatim. If the page is a generic/empty reference stub, say so.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-admin-signout" + } + ], + "resultChars": 556 } ] }, @@ -1708,7 +1808,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "The assistant correctly identified the root cause as orders missing from the supabase_realtime publication despite SUBSCRIBED succeeding, fixed it with ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, and did not weaken RLS/policies or disrupt courier_locations. It checked RLS/grants but did not blame them or change them." + "judgeNotes": "The assistant correctly identifies the silent subscription root cause as public.orders missing from the supabase_realtime publication, explains why SUBSCRIBED can still occur without INSERT delivery, and fixes specifically with ALTER PUBLICATION supabase_realtime ADD TABLE public.orders. It preserves courier_locations, RLS, policies, and does not blame client/RLS/networking or weaken security. Extra verification/notes do not violate the rubric." } ], "skills": { @@ -1721,7 +1821,32 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{searchDocs(query:\"postgres changes enable realtime publication supabase_realtime add table\", limit:4){nodes{title href content}}}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", + "title": "Subscribing to Database Changes" + }, + { + "url": "https://supabase.com/docs/guides/realtime/postgres-changes", + "title": "Postgres Changes" + }, + { + "url": "https://supabase.com/docs/guides/realtime/concepts", + "title": "Realtime Concepts" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", + "title": "Backup and Restore using the CLI" + } + ], + "resultChars": 95291 + } + ] }, "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", @@ -1752,17 +1877,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform as affected and described the 8 HTTP 503 gateway failures recurring across the morning of 2026-04-28 from 07:00Z to 12:00Z, distinguishing them from unrelated billing-webhook 503s." + "judgeNotes": "Identified image-transform as the affected function and described the recurring 503 pattern across the morning of 2026-04-28, covering all 8 gateway failures from 07:00Z through 12:00Z with retry behavior." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "The assistant clearly attributes the recurring image-transform 503s to the gateway/Edge platform layer, not function code, and grounds this in valid observations: gateway 503s had no matching edge-function invocations, nearby invocations succeeded, deployment/version stayed unchanged, and avatar-upload's function-level 500 is distinguished as a separate class of error." + "judgeNotes": "The assistant clearly attributes the recurring image-transform 503s to the gateway/platform layer before function code ran, not to application code. This is grounded in valid observations: 503s appear only in gateway/API logs with no corresponding edge-function execution logs, nearby successful invocations returned 200, deployment/version was unchanged, and it distinguishes these gateway 503s from avatar-upload's function-level 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended concrete next steps, including opening/support escalation to the platform with the specific time window and evidence, investigating the scheduled caller, adding jitter/retries, and separately triaging a code-path error." + "judgeNotes": "The assistant recommended multiple concrete next steps, including querying specific logs for the full window, checking edge-log metadata for worker/resource-limit causes, pinning dependencies, triaging a separate function error, and opening a support ticket citing correlated gateway 503 incidents." } ], "skills": { @@ -1833,7 +1958,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Diagnosed RLS enabled with no policies causing deny-all Data API results, kept RLS enabled, and created authenticated-only owner-scoped SELECT and INSERT policies using auth.uid() with WITH CHECK for inserts. Did not use permissive policies or disable RLS." + "judgeNotes": "Diagnosed RLS enabled with no policies as default-deny causing Data API empty results; kept RLS enabled; created authenticated SELECT policy with user_id = auth.uid() and authenticated INSERT policy with WITH CHECK enforcing user_id = auth.uid()." } ], "skills": { @@ -1842,51 +1967,12 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"row level security policy auth.uid() select insert performance index\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0003_auth_rls_initplan", - "title": "Database Advisor: Lint 0003_auth_rls_initplan" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/rls-performance-and-best-practices-Z5Jjwv", - "title": "RLS Performance and Best Practices" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" - } - ], - "resultChars": 67906 - }, - { - "source": "web_fetch", - "query": "List any entries tagged breaking-change related to RLS, row level security, policies, the Data API / PostgREST, grants, or auth.uid(). Just list titles, dates and one-line summaries.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 1403 - } - ] + "calls": [] }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", @@ -1937,7 +2023,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "Avatar migration was applied through `supabase db push` in action #21, with output showing `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration was reconciled by adding the missing local migration file `supabase/migrations/20240115000000_add_profile_bio.sql` in action #18, after which `supabase migration list` showed local/remote alignment and the push proceeded. I saw read-only psql inspections and a rolled-back verification schema, but no prohibited workaround or direct application of the avatar migration outside the CLI." + "judgeNotes": "Applied avatar_url via `supabase db push` in action #21, with output showing `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` Reconciled the orphan bio migration by adding the local file `supabase/migrations/20240115000000_add_profile_bio.sql` (#14), after which `supabase migration list` showed local and remote aligned (#18) and the push succeeded. No disallowed direct SQL mutation or prepared-statement workaround was used." } ], "skills": { @@ -1950,7 +2036,36 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"migration history out of sync remote migration versions not found in local migrations directory repair\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations", + "title": "Database Migrations" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-migration-repair", + "title": "Repair the migration history table" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting", + "title": "Troubleshooting" + }, + { + "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", + "title": "Multiple SSO Providers" + } + ], + "resultChars": 69700 + } + ] }, "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", @@ -2007,8 +2122,7 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { @@ -2085,7 +2199,8 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { @@ -2128,7 +2243,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 5a5d5991-7eae-49b4-8f13-2487fba8366c, signUp returned {\"userId\":\"5a5d5991-7eae-49b4-8f13-2487fba8366c\"}" + "notes": "db user 44b5425a-c28d-45d1-b278-6222633561a8, signUp returned {\"userId\":\"44b5425a-c28d-45d1-b278-6222633561a8\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -2143,7 +2258,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"5a5d5991-7eae-49b4-8f13-2487fba8366c\"}" + "notes": "{\"userId\":\"44b5425a-c28d-45d1-b278-6222633561a8\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -2207,7 +2322,7 @@ { "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "found 3 rows" + "notes": "found 2 rows" }, { "name": "row level security is enabled on todos", @@ -2225,7 +2340,7 @@ { "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "3 rows" + "notes": "2 rows" } ], "skills": { @@ -2284,32 +2399,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{searchDocs(query:\"declarative database schemas migration workflow db diff\", limit:4){nodes{title href content}}}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-db-pull", - "title": "Pull schema from the remote database" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" - } - ], - "resultChars": 62949 - } - ] + "calls": [] }, "prompt": "Add a description text column to the `products` table in my local Supabase stack", "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", @@ -2354,7 +2444,7 @@ { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 13) from the queue" + "notes": "function removed the seeded message (id 3) from the queue" } ], "skills": { @@ -2365,67 +2455,55 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"pg_cron schedule job every minute queues pgmq send\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Queues create queue pgmq send read pop delete archive\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, { "url": "https://supabase.com/docs/guides/queues/pgmq", "title": "PGMQ Extension" }, { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" }, { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" } ], - "resultChars": 69432 + "resultChars": 36950 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"Supabase Queues create queue read delete messages edge function\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"pg_cron schedule job every minute cron.schedule\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" }, { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" }, { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" + "url": "https://supabase.com/docs/guides/auth/auth-hooks/send-sms-hook", + "title": "Send SMS Hook" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pgmq", - "title": "pgmq: Queues" + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" }, { - "url": "https://supabase.com/docs/guides/functions/architecture", - "title": "Edge Functions Architecture" + "url": "https://supabase.com/docs/guides/auth/auth-hooks/send-email-hook", + "title": "Send Email Hook" } ], - "resultChars": 55770 + "resultChars": 83290 } ] }, @@ -2707,37 +2785,37 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" + "notes": "status 401: {\"error\":\"Missing credentials\"}" }, { "name": "user with JWT reads only their own rows", - "passed": false, - "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" + "passed": true, + "notes": "status 200: [{\"user_id\":\"ca5258e0-edbf-4a7e-bb54-7a1e7d854054\",\"metric\":\"steps_a_msszba7x\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", - "passed": false, - "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" + "passed": true, + "notes": "status 200: [{\"user_id\":\"ca5258e0-edbf-4a7e-bb54-7a1e7d854054\",\"metric\":\"steps_a_msszba7x\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", - "passed": false, - "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" + "passed": true, + "notes": "status 200: [{\"user_id\":\"fbe739e3-0344-4ef2-b33f-de6cb5f455e4\",\"metric\":\"steps_b_msszba7x\",\"value\":222}]" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" + "notes": "status 401: {\"error\":\"Missing credentials\"}" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" + "notes": "status 401: {\"error\":\"Invalid or expired access token\"}" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 502: { \"message\":\"An invalid response was received from the upstream server\" }" + "notes": "status 401: {\"error\":\"Missing credentials\"}" }, { "name": "implementation uses @supabase/server", @@ -2750,36 +2828,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge function environment variables SUPABASE_SERVICE_ROLE_KEY SB_SECRET_KEY publishable\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" - } - ], - "resultChars": 94601 - } - ] + "calls": [] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", @@ -2824,7 +2873,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019fdc8e-c692-723e-917b-7953678ca0d1/receipt-alpha.pdf, 019fdc8e-c692-723e-917b-7953678ca0d1/receipt-beta.pdf" + "notes": "saw: 01a00060-c8dd-7078-bec6-950dfbfd8a57/receipt-alpha.pdf, 01a00060-c8dd-7078-bec6-950dfbfd8a57/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -2845,7 +2894,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Meets all requirements: private user-files bucket, authenticated owner-scoped SELECT and INSERT policies with WITH CHECK, RLS kept enabled, and supabase-js createSignedUrl/createSignedUrls with expiry for temporary sharing." + "judgeNotes": "Meets rubric: private user-files bucket, authenticated owner-scoped SELECT and INSERT WITH CHECK policies on storage.objects, RLS kept enabled, and supabase-js createSignedUrl with expiry for temporary sharing." } ], "skills": { @@ -2853,7 +2902,60 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{\n searchDocs(query: \"storage RLS policy restrict users to their own folder user id\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + }, + { + "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", + "title": "Storage Helper Functions" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" + }, + { + "url": "https://supabase.com/docs/guides/storage/security/ownership", + "title": "Ownership" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", + "title": "Column Level Security" + } + ], + "resultChars": 24828 + }, + { + "source": "search_docs", + "query": "{\n searchDocs(query: \"createSignedUrl expiring share private file\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" + }, + { + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" + }, + { + "url": "https://supabase.com/docs/reference/dart/file-buckets-createsignedurl" + }, + { + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" + }, + { + "url": "https://supabase.com/docs/reference/csharp/storage-from-createsignedurl", + "title": "From().CreateSignedUrl()" + } + ], + "resultChars": 3603 + } + ] }, "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", @@ -2885,17 +2987,17 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "3 file(s): supabase/tests/database/02_posts_tenant_isolation.test.sql, supabase/tests/database/03_memberships_and_writes.test.sql, supabase/tests/database/01_notes_tenant_isolation.test.sql" + "notes": "1 file(s): supabase/tests/tenant_isolation.test.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "4 passed, 2 failed" + "notes": "20 passed, 5 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as the broken tenant isolation policy, explains that authenticated members can read posts from orgs they are not members of, and grounds this in pgTAP/direct test results. It also distinguishes `notes` as passing isolation." + "judgeNotes": "Correctly identifies `posts` as the broken tenant isolation policy, explains that any authenticated user with any membership can read all posts due to missing `m.org_id = posts.org_id`, and grounds the conclusion in pgTAP failures and direct reproduction. It also correctly states `notes` is isolated." } ], "skills": { @@ -2970,7 +3072,40 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"semantic search pgvector match_document_sections RLS embedding gte-small\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" + }, + { + "url": "https://supabase.com/docs/guides/ai/hybrid-search", + "title": "Hybrid search" + }, + { + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pgvector", + "title": "pgvector: Embeddings and vector similarity" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", + "title": "Semantic Search" + } + ], + "resultChars": 80747 + } + ] }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", @@ -3004,12 +3139,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "judgeNotes": "Supabase scrape uses HTTPS, correct metrics path, Basic Auth with password_file, targets a supabase.co project ref, preserves the app job, and docker-compose mounts the secrets directory containing the password file path." + "judgeNotes": "Meets requirements: preserves app scrape, adds HTTPS Supabase Metrics API target on supabase.co with correct path, uses HTTP Basic Auth with password_file, and docker-compose mounts the secrets directory containing that password_file path." }, { "name": "documented live deployment and verification steps", "passed": true, - "judgeNotes": "README includes Secret API key creation, matching secret file path, Compose restart/up and reload guidance, plus concrete verification via Prometheus targets, PromQL API, and Grafana dashboard checks." + "judgeNotes": "README includes Secret API key creation, secret file path, reload/restart steps, and concrete Prometheus target verification." } ], "skills": { @@ -3020,7 +3155,7 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"Prometheus metrics endpoint scrape project metrics observability\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"Prometheus metrics endpoint project observability scrape\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -3031,6 +3166,10 @@ "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", "title": "Vendor-agnostic Metrics API setup" }, + { + "url": "https://supabase.com/docs/reference/api/v1-scrape-project-metrics", + "title": "Scrape a project's metrics" + }, { "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", "title": "Metrics API with Prometheus & Grafana (self-hosted)" @@ -3040,13 +3179,13 @@ "title": "Metrics API with Grafana Cloud" } ], - "resultChars": 19943 + "resultChars": 20571 } ] }, "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-5-no-skills/deploy-database-001-prometheus-metrics.json" }, { @@ -3097,7 +3236,7 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"edge functions environment variables secrets manage\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"edge function secrets environment variables Deno.env.get\", limit: 4) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -3105,23 +3244,19 @@ "title": "Environment Variables" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", + "title": "Inspecting edge function environment variables" }, { - "url": "https://supabase.com/docs/reference/cli/supabase-secrets", - "title": "Manage Supabase secrets" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", "title": "Automatic embeddings" } ], - "resultChars": 61195 + "resultChars": 52838 } ] }, @@ -3233,7 +3368,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "The answer diagnoses the original flow as only soft-deleting public.profiles and not touching auth.users/sessions/refresh tokens. It implements real revocation by banning/soft-deleting the auth user and deleting refresh_tokens/sessions, plus RLS gating. It consistently explains JWT access tokens remain cryptographically valid until expiry for local validation, while its implemented RLS/session-existence check closes the Data API window. It also correctly distinguishes publishable frontend keys under anon/authenticated + RLS from secret/service_role backend-only keys that bypass RLS." + "judgeNotes": "Meets the rubric. It identifies the original soft-delete-only flow, implements meaningful auth-side revocation by banning/marking deleted and deleting sessions/refresh tokens while enforcing the deleted flag in RLS, and explains the remaining stateless JWT caveat consistently: Data API window is closed by RLS, but purely local JWT validation would still accept until expiry. It correctly distinguishes publishable frontend/RLS-protected keys from secret server-only/RLS-bypassing keys and does not recommend exposing secrets." } ], "skills": { @@ -3244,55 +3379,7 @@ "calls": [ { "source": "search_docs", - "query": "{\n banned: searchDocs(query: \"ban user banned_until revoke session delete user admin API\", limit: 4) {\n nodes { title href content }\n }\n keys: searchDocs(query: \"publishable key secret key legacy anon service_role API keys migration RLS\", limit: 6) {\n nodes { title href content }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", - "title": "Build a User Management App with SvelteKit" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-solidjs", - "title": "Build a User Management App with SolidJS" - } - ], - "resultChars": 245869 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"publishable key secret key API keys\", limit: 3) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"publishable key secret key API keys migration anon service_role\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -3306,1316 +3393,216 @@ { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", "title": "New API Keys and Asymmetric Authentication" - } - ], - "resultChars": 61983 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"sb_secret sb_publishable new API keys migration\", limit: 2) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - } - ], - "resultChars": 29035 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Understanding API keys publishable key anon role RLS\", limit: 1) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - } - ], - "resultChars": 13676 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"secret key bypasses Row Level Security service_role postgres role\", limit: 2) { nodes { title href } } }", - "hasContent": false, - "pages": [ + "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", + "title": "Supabase for Platforms" + }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", + "title": "Build a Social Auth App with Expo React Native" } ], - "resultChars": 329 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"ban user ban_duration admin updateUserById\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-updateuserbyid" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-admin-updateuserbyid" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-updateuser" - } - ], - "resultChars": 3573 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"deleteUser soft delete admin\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-admin-deleteuser", - "title": "deleteUser()" - } - ], - "resultChars": 1237 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"disable legacy anon service_role key dashboard rotate JWT secret\", limit: 2) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/rotating-anon-service-and-jwt-secrets-1Jq6yd", - "title": "Rotating Anon, Service, and JWT Secrets" - } - ], - "resultChars": 34590 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"JWT expiry access token lifetime sessions\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" - } - ], - "resultChars": 59617 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"auth.admin.signOut revoke sessions scope global\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/signout", - "title": "Signing out" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signout" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-signout" - } - ], - "resultChars": 5307 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"legacy API keys deprecation end of 2026 removal timeline\", limit: 2) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - } - ], - "resultChars": 45967 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"banned_until auth.users table columns\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" - } - ], - "resultChars": 31852 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"delete account user edge function service role\", limit: 2) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" - } - ], - "resultChars": 20425 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"do not modify auth schema managed by Supabase\", limit: 2) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/prisma/prisma-troubleshooting", - "title": "Troubleshooting prisma errors" - }, - { - "url": "https://supabase.com/docs/guides/auth/architecture", - "title": "Auth architecture" - } - ], - "resultChars": 16231 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"verifying a JWT from Supabase getClaims asymmetric local verification\", limit: 2) { nodes { title href } subsections: nodes { title href } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" - } - ], - "resultChars": 490 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"JSON Web Token JWT verifying revocation\", limit: 1) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" - } - ], - "resultChars": 14157 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"banned user cannot sign in ban\", limit: 3) { nodes { title href } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", - "title": "Passwordless email logins" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-identity-linking", - "title": "Identity Linking" - } - ], - "resultChars": 392 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"cannot delete user foreign key constraint troubleshooting\", limit: 3) { nodes { title href } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0021_fkey_to_auth_unique", - "title": "Database Advisor: Lint 0021_fkey_to_auth_unique" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/cascade-deletes", - "title": "Cascade Deletes" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - } - ], - "resultChars": 468 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"user management managing users admin api\", limit: 3) { nodes { title href } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-refine", - "title": "Build a User Management App with Refine" - }, - { - "url": "https://supabase.com/docs/guides/auth/users", - "title": "Users" - } - ], - "resultChars": 406 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Users guide auth users table permanent anonymous\", limit: 1) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/users", - "title": "Users" - } - ], - "resultChars": 11521 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"managing user data profiles trigger accessing user data\", limit: 1) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", - "title": "Build a User Management App with Angular" - } - ], - "resultChars": 34746 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"signOut jwt scope admin server side revoke refresh tokens\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-signout" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" - }, - { - "url": "https://supabase.com/docs/reference/javascript/oauth-server-revokegrant" - } - ], - "resultChars": 38073 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"JWT claims reference session_id is_anonymous fields\", limit: 1) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" - } - ], - "resultChars": 12679 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"shouldSoftDelete\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", - "title": "Deleting data and dropping objects safely" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/soft-deletes-with-supabase-js", - "title": "Implementing soft deletes with supabase-js" - } - ], - "resultChars": 12330 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"service role key never expose server side edge function admin client\", limit: 2) { nodes { title href } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - } - ], - "resultChars": 326 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Data API PostgREST how requests are authorized JWT role\", limit: 2) { nodes { title href } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles", - "title": "Postgres Roles" - } - ], - "resultChars": 273 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"access token expiry setting jwt_exp configuration auth\", limit: 2) { nodes { title href } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V", - "title": "Should I set a shorter Max-Age parameter on the cookies?" - } - ], - "resultChars": 358 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"user_banned error code\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - } - ], - "resultChars": 11900 - }, - { - "source": "search_docs", - "query": "{ error(code: \"user_banned\", service: AUTH) { code httpStatusCode message } }", - "hasContent": false, - "pages": [], - "resultChars": 280 - }, - { - "source": "search_docs", - "query": "{ a: error(code: \"session_not_found\", service: AUTH) { code message } b: error(code: \"user_not_found\", service: AUTH) { code message } c: error(code: \"refresh_token_not_found\", service: AUTH) { code message } d: error(code: \"refresh_token_already_used\", service: AUTH) { code message } e: error(code: \"session_expired\", service: AUTH) { code message } }", - "hasContent": false, - "pages": [], - "resultChars": 971 - } - ] - }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "attempts": 2, - "sourcePath": "claude-code-opus-5-no-skills/investigate-auth-001-deleted-user-access.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "investigate-realtime-001-subscribed-no-events", - "stage": "investigate", - "product": [ - "realtime", - "database" - ], - "topic": [ - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "orders table added to supabase_realtime publication", - "passed": true - }, - { - "name": "courier_locations still in supabase_realtime publication", - "passed": true - }, - { - "name": "publication still publishes INSERT events", - "passed": true - }, - { - "name": "RLS still enabled on orders", - "passed": true - }, - { - "name": "staff can still read orders through RLS", - "passed": true, - "notes": "authenticated sees 2 of 2 orders" - }, - { - "name": "diagnosed missing publication membership", - "passed": true, - "judgeNotes": "The answer correctly identifies that the channel can reach SUBSCRIBED while INSERT events are silent because public.orders was missing from the supabase_realtime publication. It fixes exactly that with ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, preserves courier_locations, keeps RLS/policies intact, and explicitly does not blame RLS or client code." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", - "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/investigate-realtime-001-subscribed-no-events.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "investigate-reliability-003-edge-function-5xx-correlation", - "stage": "investigate", - "product": [ - "edge-functions" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "identified image-transform and the recurring 503 pattern", - "passed": true, - "judgeNotes": "The assistant clearly identified `image-transform` as the affected function and described the recurring pattern of eight HTTP 503 gateway failures spread through the morning of 2026-04-28, while distinguishing them from older billing-webhook 503s." - }, - { - "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": true, - "judgeNotes": "Attributes the recurring image-transform 503s to the gateway/platform layer, not function code, and grounds this in valid observations: 503s only in gateway logs with no runtime invocations, nearby successful invocations, unchanged deployment/version, and distinction from avatar-upload's function-level 500." - }, - { - "name": "recommended a concrete next step", - "passed": true, - "judgeNotes": "The assistant recommended multiple concrete next steps: investigate half-hour scheduled jobs/concurrency, query function_edge_logs for status>=500, check Edge Function limits/metrics, open a support ticket with timestamps, and examine avatar-upload error output." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", - "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/investigate-reliability-003-edge-function-5xx-correlation.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "resolve-dataapi-001-empty-results", - "stage": "resolve", - "product": [ - "data-api", - "database", - "auth" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS still enabled on bookmarks", - "passed": true - }, - { - "name": "user A reads own bookmarks", - "passed": true - }, - { - "name": "user B cannot read user A bookmarks", - "passed": true - }, - { - "name": "anon reads no bookmarks", - "passed": true - }, - { - "name": "user A can save a new bookmark", - "passed": true - }, - { - "name": "user B cannot insert a bookmark as user A", - "passed": true - }, - { - "name": "diagnosed RLS and added owner-scoped policies", - "passed": true, - "judgeNotes": "Diagnosed RLS enabled with zero policies as the cause of empty Data API results, kept RLS enabled, and created authenticated-only owner-scoped SELECT and INSERT policies using auth.uid() with WITH CHECK for inserts." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", - "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/resolve-dataapi-001-empty-results.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "resolve-database-001-migration-history-mismatch", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "the avatar_url column is applied on the hosted profiles table", - "passed": true - }, - { - "name": "migration 20240220000000 is recorded in the remote history", - "passed": true - }, - { - "name": "remote migration history matches local migration files", - "passed": true - }, - { - "name": "local migrations are a valid reconciled sequence", - "passed": true - }, - { - "name": "production profile data is intact (not reset)", - "passed": true - }, - { - "name": "the avatar migration and history reconciliation were done via the Supabase CLI", - "passed": true, - "judgeNotes": "Avatar migration was applied through the Supabase CLI with `supabase db push` in action #19, showing `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration was reconciled by adding the local file `20240115000000_add_profile_bio.sql` in action #16, after which Supabase CLI migration list showed it matched remote (#17) and the subsequent `db push` succeeded. No disallowed direct-SQL mutation or prepared-statement workaround was used." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", - "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/resolve-database-001-migration-history-mismatch.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "resolve-performance-001-slow-query-cpu-spike", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "observability", - "sql" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "inspected pg_stat_statements for query performance", - "passed": true - }, - { - "name": "ran EXPLAIN on the expensive query", - "passed": true - }, - { - "name": "created index covering user_id and created_at", - "passed": true - }, - { - "name": "query plan uses an index and avoids sequential scan", - "passed": true, - "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_desc_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" - }, - { - "name": "inserts still work", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", - "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/resolve-performance-001-slow-query-cpu-spike.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "resolve-security-002-rls-cross-tenant-leak", - "stage": "resolve", - "product": [ - "database", - "auth" - ], - "topic": [ - "rls", - "security" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS enabled on notes", - "passed": true - }, - { - "name": "tenant A sees only org A notes", - "passed": true - }, - { - "name": "tenant B cannot read org A notes", - "passed": true - }, - { - "name": "tenant A author can update own note", - "passed": true - }, - { - "name": "tenant B cannot update org A note", - "passed": true - }, - { - "name": "tenant B author can delete own note", - "passed": true - }, - { - "name": "tenant B cannot delete org A note", - "passed": true - }, - { - "name": "tenant A can insert note in own org", - "passed": true - }, - { - "name": "tenant B cannot insert into org A", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/resolve-security-002-rls-cross-tenant-leak.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-auth-001-email-password-flow", - "stage": "build", - "product": [ - "auth", - "database" - ], - "topic": [ - "sdk", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "auth module loads and the driver completes", - "passed": true, - "notes": "driver produced a result" - }, - { - "name": "signUp creates the account and returns its user id", - "passed": true, - "notes": "db user 0ac704f8-596e-4f98-84c9-00595301873d, signUp returned {\"userId\":\"0ac704f8-596e-4f98-84c9-00595301873d\"}" - }, - { - "name": "signup metadata reaches the profile (display name)", - "passed": true, - "notes": "profiles.display_name = \"Alex Doe\"" - }, - { - "name": "wrong password is rejected gracefully (no throw, no session)", - "passed": true, - "notes": "{\"error\":\"Invalid login credentials\"}" - }, - { - "name": "signIn with the right password returns the user id", - "passed": true, - "notes": "{\"userId\":\"0ac704f8-596e-4f98-84c9-00595301873d\"}" - }, - { - "name": "getMyProfile returns the signed-in user's profile", - "passed": true, - "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" - }, - { - "name": "app code does not use the secret / service-role key", - "passed": true, - "notes": "no secret-key references found" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/src/auth.mjs" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", - "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-auth-001-email-password-flow.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-cli-001-bootstrap-app", - "stage": "build", - "product": [ - "database", - "data-api" - ], - "topic": [ - "migrations", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "supabase project initialised (supabase/config.toml exists)", - "passed": true - }, - { - "name": "todos table is created by a migration file", - "passed": true - }, - { - "name": "todos table exists with at least 2 seeded rows", - "passed": true, - "notes": "found 2 rows" - }, - { - "name": "row level security is enabled on todos", - "passed": true - }, - { - "name": "a SELECT policy targets the authenticated role", - "passed": true - }, - { - "name": "REST API returns no todos to anonymous requests", - "passed": true, - "notes": "0 rows" - }, - { - "name": "REST API returns the todos to authenticated requests", - "passed": true, - "notes": "2 rows" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-cli-001-bootstrap-app.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-cli-002-declarative-schema", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "declarative-schema", - "migrations" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "supabase db diff used to generate the migration", - "passed": true - }, - { - "name": "schema file updated to include description column", - "passed": true - }, - { - "name": "a new migration was generated for the change", - "passed": true - }, - { - "name": "description column exists in the live database", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "Add a description text column to the `products` table in my local Supabase stack", - "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-cli-002-declarative-schema.json" - }, - { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "eval": "build-cli-003-pg-cron-queue-workflow", - "stage": "build", - "product": [ - "database", - "edge-functions", - "cron", - "queues" - ], - "topic": [ - "sql", - "sdk" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" - }, - { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 0 -> 1" - }, - { - "name": "process-tasks function drains the queue", - "passed": true, - "notes": "function removed the seeded message (id 3) from the queue" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"cron.schedule pgmq send queue example\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - } - ], - "resultChars": 35564 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Functions read from queue pgmq delete message worker\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/recursive-functions", - "title": "Recursive / Nested Function Calls" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", - "title": "Backup and Restore using the CLI" - } - ], - "resultChars": 93380 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Functions default environment variables SUPABASE_SERVICE_ROLE_KEY SUPABASE_URL automatically available\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - } - ], - "resultChars": 28165 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Queues schedule cron job to process messages Edge Function example\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-hooks/send-email-hook", - "title": "Send Email Hook" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/slack-bot-mention", - "title": "Slack Bot Mention Edge Function" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - } - ], - "resultChars": 81933 + "resultChars": 166856 } ] }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-cli-003-pg-cron-queue-workflow.json" + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-opus-5-no-skills/investigate-auth-001-deleted-user-access.json" }, { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", + "experiment": "claude-code-opus-5-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", + "modelId": "claude-opus-5", "reasoningEffort": "high" }, - "eval": "build-dataapi-001-relational-report", - "stage": "build", + "eval": "investigate-realtime-001-subscribed-no-events", + "stage": "investigate", "product": [ - "data-api", + "realtime", "database" ], "topic": [ "sdk" ], "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", + "interface": "mcp", "passed": true, "checks": [ { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" + "name": "orders table added to supabase_realtime publication", + "passed": true }, { - "name": "report numbers match the database (per customer, sorted)", + "name": "courier_locations still in supabase_realtime publication", + "passed": true + }, + { + "name": "publication still publishes INSERT events", + "passed": true + }, + { + "name": "RLS still enabled on orders", + "passed": true + }, + { + "name": "staff can still read orders through RLS", "passed": true, - "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + "notes": "authenticated sees 2 of 2 orders" }, { - "name": "tables stay locked down (publishable key reads nothing)", + "name": "diagnosed missing publication membership", "passed": true, - "notes": "publishable read errored: permission denied for table customers" + "judgeNotes": "Diagnosed the issue as orders missing from supabase_realtime despite channel SUBSCRIBED, applied ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, and did not weaken RLS/policies or disrupt courier_locations." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", + "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-5-no-skills/investigate-realtime-001-subscribed-no-events.json" + }, + { + "experiment": "claude-code-opus-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-5", + "reasoningEffort": "high" + }, + "eval": "investigate-reliability-003-edge-function-5xx-correlation", + "stage": "investigate", + "product": [ + "edge-functions" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "identified image-transform and the recurring 503 pattern", + "passed": true, + "judgeNotes": "Assistant identified image-transform and described the recurring 8 gateway-only HTTP 503 failures across 07:00Z-12:00Z on 2026-04-28, while distinguishing billing-webhook as unrelated." }, { - "name": "implementation uses @supabase/supabase-js", + "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "notes": "imports found in: app/report.mjs" + "judgeNotes": "The assistant clearly attributes the recurring image-transform 503s to the gateway/platform layer, stating the requests never ran the function and were gateway-only. This is grounded in valid observations: no corresponding edge-function invocation records for the 503 windows while nearby invocations succeeded, unchanged deployment/version across the window, and distinction from avatar-upload's in-function 500." }, { - "name": "report queries via the Data API, not raw SQL", + "name": "recommended a concrete next step", "passed": true, - "notes": "no psql / raw Postgres driver usage found" + "judgeNotes": "The assistant provided multiple concrete next steps, including checking scheduled half-hour jobs, obtaining 503 error bodies and edge-function resource metrics for specific timestamps, pulling the avatar-upload stack trace, and confirming the upload routing path." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] + "available": [], + "loaded": [] }, "docs": { "calls": [] }, - "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", - "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", + "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-dataapi-001-relational-report.json" + "sourcePath": "claude-code-opus-5-no-skills/investigate-reliability-003-edge-function-5xx-correlation.json" }, { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", + "experiment": "claude-code-opus-5-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", + "modelId": "claude-opus-5", "reasoningEffort": "high" }, - "eval": "build-dataapi-002-restock-alert-report", - "stage": "build", + "eval": "resolve-dataapi-001-empty-results", + "stage": "resolve", "product": [ "data-api", - "database" + "database", + "auth" ], "topic": [ + "rls", "sdk" ], "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", + "interface": "mcp", "passed": true, "checks": [ { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" + "name": "RLS still enabled on bookmarks", + "passed": true }, { - "name": "alerts match the database (below threshold, sorted)", - "passed": true, - "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + "name": "user A reads own bookmarks", + "passed": true }, { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table inventory" + "name": "user B cannot read user A bookmarks", + "passed": true }, { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/restock.mjs" + "name": "anon reads no bookmarks", + "passed": true }, { - "name": "report queries via the Data API, not raw SQL", + "name": "user A can save a new bookmark", + "passed": true + }, + { + "name": "user B cannot insert a bookmark as user A", + "passed": true + }, + { + "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "notes": "no psql / raw Postgres driver usage found" + "judgeNotes": "Diagnosed RLS enabled with no policies as default-deny for Data API, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] + "available": [], + "loaded": [] }, "docs": { "calls": [] }, - "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", - "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", + "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-dataapi-002-restock-alert-report.json" + "sourcePath": "claude-code-opus-5-no-skills/resolve-dataapi-001-empty-results.json" }, { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", + "experiment": "claude-code-opus-5-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", + "modelId": "claude-opus-5", "reasoningEffort": "high" }, - "eval": "build-database-001-migrate-postgres-to-supabase", - "stage": "build", + "eval": "resolve-database-001-migration-history-mismatch", + "stage": "resolve", "product": [ "database" ], @@ -4627,402 +3614,170 @@ "passed": true, "checks": [ { - "name": "all 3 tables exist (teams, members, tasks)", + "name": "the avatar_url column is applied on the hosted profiles table", "passed": true }, { - "name": "row counts match (teams=5, members=10, tasks=13)", + "name": "migration 20240220000000 is recorded in the remote history", "passed": true }, { - "name": "foreign key constraints survived the restore", + "name": "remote migration history matches local migration files", "passed": true }, { - "name": "tasks_team_status_idx index survived the restore", + "name": "local migrations are a valid reconciled sequence", "passed": true }, { - "name": "sequences synced (next insert won't conflict with existing IDs)", + "name": "production profile data is intact (not reset)", "passed": true + }, + { + "name": "the avatar migration and history reconciliation were done via the Supabase CLI", + "passed": true, + "judgeNotes": "Avatar migration was applied through `supabase db push` in action #19, with output `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration was reconciled by adding the matching local file `supabase/migrations/20240115000000_add_profile_bio.sql` in #17, after which CLI `supabase migration list`/`supabase db push --dry-run` showed histories aligned and only avatar pending. No disallowed workaround was used; psql commands were read-only inspection." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] + "available": [], + "loaded": [] }, "docs": { "calls": [] }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", + "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-database-001-migrate-postgres-to-supabase.json" + "sourcePath": "claude-code-opus-5-no-skills/resolve-database-001-migration-history-mismatch.json" }, { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", + "experiment": "claude-code-opus-5-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", + "modelId": "claude-opus-5", "reasoningEffort": "high" }, - "eval": "build-functions-004-service-role-bypass", - "stage": "build", + "eval": "resolve-performance-001-slow-query-cpu-spike", + "stage": "resolve", "product": [ - "edge-functions", - "auth", "database" ], "topic": [ - "rls", - "security", - "sdk" + "observability", + "sql" ], "suite": "benchmark", "interface": "mcp", "passed": true, "checks": [ { - "name": "rejects missing auth", - "passed": true, - "notes": "status=401" + "name": "inspected pg_stat_statements for query performance", + "passed": true }, { - "name": "user A reads own note", - "passed": true, - "notes": "status=200" + "name": "ran EXPLAIN on the expensive query", + "passed": true }, { - "name": "reads only with the caller's JWT", - "passed": true, - "notes": "bearer_tokens=2, all_match=true" + "name": "created index covering user_id and created_at", + "passed": true }, { - "name": "user A cannot force-read user B note", + "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "status=200" + "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_desc_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { - "name": "user B cannot force-read user A note", - "passed": true, - "notes": "status=200" + "name": "inserts still work", + "passed": true } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] + "available": [], + "loaded": [] }, "docs": { "calls": [] }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", + "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", + "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-functions-004-service-role-bypass.json" + "sourcePath": "claude-code-opus-5-no-skills/resolve-performance-001-slow-query-cpu-spike.json" }, { - "experiment": "claude-code-sonnet-5", - "experimentSuite": "benchmark", + "experiment": "claude-code-opus-5-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", + "modelId": "claude-opus-5", "reasoningEffort": "high" }, - "eval": "build-functions-005-dual-auth-user-secret", - "stage": "build", + "eval": "resolve-security-002-rls-cross-tenant-leak", + "stage": "resolve", "product": [ - "edge-functions", - "auth", - "database" + "database", + "auth" ], "topic": [ - "sdk", "rls", "security" ], "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", + "interface": "mcp", "passed": true, "checks": [ { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"9dfb45f7-a584-480f-9567-31a82085a4db\",\"metric\":\"steps_a_msj1bz52\",\"value\":111}]}" - }, - { - "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"9dfb45f7-a584-480f-9567-31a82085a4db\",\"metric\":\"steps_a_msj1bz52\",\"value\":111}]}" + "name": "RLS enabled on notes", + "passed": true }, { - "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"7ef32f13-71a0-4757-84bc-00f9632c3443\",\"metric\":\"steps_b_msj1bz52\",\"value\":222}]}" + "name": "tenant A sees only org A notes", + "passed": true }, { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "name": "tenant B cannot read org A notes", + "passed": true }, { - "name": "rejects an unverified (forged) user token", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "name": "tenant A author can update own note", + "passed": true }, { - "name": "a user token in the apikey slot is not treated as the service key", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "name": "tenant B cannot update org A note", + "passed": true }, { - "name": "implementation uses @supabase/server", - "passed": true, - "notes": "imports @supabase/server / withSupabase" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Functions environment variables SUPABASE_URL SUPABASE_SERVICE_ROLE_KEY SUPABASE_ANON_KEY secret key\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/storage-caching", - "title": "Integrating with Supabase Storage" - } - ], - "resultChars": 36932 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"new API keys publishable secret key edge functions verify_jwt\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - } - ], - "resultChars": 70715 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Securing Edge Functions auth modes user secret publishable multiple auth withSupabase array\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - } - ], - "resultChars": 45970 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Authorization headers edge functions apikey header verify_jwt disable\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-hooks/send-email-hook", - "title": "Send Email Hook" - } - ], - "resultChars": 95133 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"combining auth modes user secret verify_jwt false config.toml example\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips" - }, - { - "url": "https://supabase.com/docs/guides/functions/deploy", - "title": "Deploy to Production" - }, - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - } - ], - "resultChars": 24076 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"@supabase/server package version pin npm install deno.json import map\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/security/npm-security", - "title": "Securing npm installs" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", - "title": "Building an MCP Server with mcp-lite" - }, - { - "url": "https://supabase.com/docs/guides/functions/dependencies", - "title": "Managing dependencies" - } - ], - "resultChars": 34326 - }, - { - "source": "web_search", - "query": "\"failed to determine entrypoint\" supabase edge-runtime worker boot error", - "pages": [], - "resultChars": 2031 - }, - { - "source": "web_search", - "query": "supabase cli issue 4190 podman edge functions serve entrypoint bind mount", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/troubleshooting", - "title": "Supabase Docs | Edge Functions Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/issues-serving-edge-functions-locally", - "title": "Supabase Docs | Troubleshooting | Issues serving Edge Functions locally" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration | Supabase Docs" - }, - { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips | Supabase Docs" - } - ], - "resultChars": 2355 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Issues serving Edge Functions locally troubleshooting entrypoint bind mount docker context\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/issues-serving-edge-functions-locally", - "title": "Issues serving Edge Functions locally" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - } - ], - "resultChars": 45231 - } - ] + "name": "tenant B author can delete own note", + "passed": true + }, + { + "name": "tenant B cannot delete org A note", + "passed": true + }, + { + "name": "tenant A can insert note in own org", + "passed": true + }, + { + "name": "tenant B cannot insert into org A", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 2, - "sourcePath": "claude-code-sonnet-5/build-functions-005-dual-auth-user-secret.json" + "docs": { + "calls": [] + }, + "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", + "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-5-no-skills/resolve-security-002-rls-cross-tenant-leak.json" }, { "experiment": "claude-code-sonnet-5", @@ -5033,57 +3788,60 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "build-storage-001-private-bucket-access", + "eval": "build-auth-001-email-password-flow", "stage": "build", "product": [ - "storage", + "auth", "database" ], "topic": [ - "rls", - "sdk" + "sdk", + "rls" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", + "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", - "passed": true + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" }, { - "name": "RLS still enabled on storage.objects", - "passed": true + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user dcf53034-7204-47f4-88a8-b9e156b1894c, signUp returned {\"userId\":\"dcf53034-7204-47f4-88a8-b9e156b1894c\"}" }, { - "name": "user A lists only own files", + "name": "signup metadata reaches the profile (display name)", "passed": true, - "notes": "saw: 019fdc8d-2b1c-70b8-a74d-7a4097cf4fa6/receipt-alpha.pdf, 019fdc8d-2b1c-70b8-a74d-7a4097cf4fa6/receipt-beta.pdf" + "notes": "profiles.display_name = \"Alex Doe\"" }, { - "name": "user B cannot read user A files", - "passed": true + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" }, { - "name": "anon reads no files", - "passed": true + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"dcf53034-7204-47f4-88a8-b9e156b1894c\"}" }, { - "name": "user A can upload into own folder", - "passed": true + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" }, { - "name": "user B cannot upload into user A folder", - "passed": true + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" }, { - "name": "configured private per-user storage access", + "name": "implementation uses @supabase/supabase-js", "passed": true, - "judgeNotes": "The answer creates a private user-files bucket, defines authenticated SELECT and INSERT policies on storage.objects scoped to bucket_id and the user's UID folder with WITH CHECK for uploads, keeps RLS enabled, avoids public/anon/service-role pitfalls, and provides supabase-js createSignedUrl with an expiry for temporary sharing." + "notes": "imports found in: app/src/auth.mjs" } ], "skills": { @@ -5096,66 +3854,12 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"storage RLS policy owner folder path user id\", limit: 5) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/ownership", - "title": "Ownership" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" - } - ], - "resultChars": 61599 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"createSignedUrl expiring share link\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" - }, - { - "url": "https://supabase.com/docs/reference/dart/file-buckets-createsignedurl" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" - }, - { - "url": "https://supabase.com/docs/reference/csharp/storage-from-createsignedurl", - "title": "From().CreateSignedUrl()" - } - ], - "resultChars": 7873 - } - ] + "calls": [] }, - "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", - "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-storage-001-private-bucket-access.json" + "sourcePath": "claude-code-sonnet-5/build-auth-001-email-password-flow.json" }, { "experiment": "claude-code-sonnet-5", @@ -5166,13 +3870,14 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "build-tests-001-rls-tenant-isolation", + "eval": "build-cli-001-bootstrap-app", "stage": "build", "product": [ - "database" + "database", + "data-api" ], "topic": [ - "tests", + "migrations", "rls" ], "suite": "benchmark", @@ -5180,19 +3885,35 @@ "passed": true, "checks": [ { - "name": "pgTAP test file(s) written under supabase/tests/", + "name": "supabase project initialised (supabase/config.toml exists)", + "passed": true + }, + { + "name": "todos table is created by a migration file", + "passed": true + }, + { + "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" + "notes": "found 2 rows" }, { - "name": "pgTAP isolation tests ran and pass", + "name": "row level security is enabled on todos", + "passed": true + }, + { + "name": "a SELECT policy targets the authenticated role", + "passed": true + }, + { + "name": "REST API returns no todos to anonymous requests", "passed": true, - "notes": "6 passed, 4 failed" + "notes": "0 rows" }, { - "name": "agent correctly identifies the posts isolation bug from test results", + "name": "REST API returns the todos to authenticated requests", "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as having a broken tenant isolation SELECT policy, specifically that members of any org can read posts from other orgs, and grounds this in pgTAP test 5 failing. It treats test results as authoritative and contrasts `posts` with `notes`, which passed isolation tests. It also notes additional membership-table issues, but does not blame `notes` instead of `posts`." + "notes": "2 rows" } ], "skills": { @@ -5208,10 +3929,10 @@ "docs": { "calls": [] }, - "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", - "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", + "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", + "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-tests-001-rls-tenant-isolation.json" + "sourcePath": "claude-code-sonnet-5/build-cli-001-bootstrap-app.json" }, { "experiment": "claude-code-sonnet-5", @@ -5222,50 +3943,92 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "build-vectors-001-rag-with-permissions", + "eval": "build-cli-002-declarative-schema", "stage": "build", "product": [ - "database", - "vectors" + "database" ], "topic": [ - "sql", - "rls" + "declarative-schema", + "migrations" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", "passed": true, "checks": [ { - "name": "document_sections.embedding is vector(384)", - "passed": true, - "notes": "vector(384)" - }, - { - "name": "HNSW index on the embedding column", - "passed": true, - "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + "name": "supabase db diff used to generate the migration", + "passed": true }, { - "name": "index operator class matches the search operator", - "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + "name": "schema file updated to include description column", + "passed": true }, { - "name": "user A search returns only own sections, best match first", + "name": "a new migration was generated for the change", "passed": true }, { - "name": "user B search returns only own sections, best match first", + "name": "description column exists in the live database", "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Add a description text column to the `products` table in my local Supabase stack", + "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5/build-cli-002-declarative-schema.json" + }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-cli-003-pg-cron-queue-workflow", + "stage": "build", + "product": [ + "database", + "edge-functions", + "cron", + "queues" + ], + "topic": [ + "sql", + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", + "passed": true, + "notes": "schedule='* * * * *', active=true" }, { - "name": "user A reads only own sections through the API", - "passed": true + "name": "cron command enqueues to the 'tasks' queue", + "passed": true, + "notes": "queue depth 1 -> 2" }, { - "name": "user A reads only own documents through the API", - "passed": true + "name": "process-tasks function drains the queue", + "passed": true, + "notes": "function removed the seeded message (id 37) from the queue" } ], "skills": { @@ -5282,50 +4045,86 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"semantic search edge functions pgvector gte-small match_document_sections\", limit: 3) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"pg_cron schedule pgmq queue example cron.schedule\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" }, { - "url": "https://supabase.com/docs/guides/database/full-text-search", - "title": "Full Text Search" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" }, { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" + "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", + "title": "pg_cron debugging guide" + }, + { + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" + }, + { + "url": "https://supabase.com/docs/guides/cron/install", + "title": "Install" } ], - "resultChars": 61699 + "resultChars": 55632 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"vector extension schema best practice extensions schema\", limit: 3) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"pgmq queues read delete message edge function example\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started", - "title": "Getting Started with Realtime" + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "title": "Consuming Supabase Queue Messages with Edge Functions" }, { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" + "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", + "title": "Backup and Restore using the CLI" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + } + ], + "resultChars": 103837 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Supabase Queues pgmq.create pgmq.send pgmq.read pgmq.delete\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + }, + { + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" + }, + { + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" } ], - "resultChars": 46198 + "resultChars": 44663 } ] }, - "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", - "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-vectors-001-rag-with-permissions.json" + "sourcePath": "claude-code-sonnet-5/build-cli-003-pg-cron-queue-workflow.json" }, { "experiment": "claude-code-sonnet-5", @@ -5336,30 +4135,44 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "deploy-database-001-prometheus-metrics", - "stage": "deploy", + "eval": "build-dataapi-001-relational-report", + "stage": "build", "product": [ + "data-api", "database" ], "topic": [ - "observability" + "sdk" ], "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "preserved existing app scrape job", - "passed": true + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" }, { - "name": "configured the Supabase Metrics API scrape correctly", + "name": "report numbers match the database (per customer, sorted)", "passed": true, - "judgeNotes": "Meets requirements: HTTPS Supabase Metrics API scrape at the correct path with project target, basic_auth using password_file, app scrape preserved, and docker-compose mounts the secrets directory containing the password file." + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" }, { - "name": "documented live deployment and verification steps", + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/report.mjs" + }, + { + "name": "report queries via the Data API, not raw SQL", "passed": true, - "judgeNotes": "README includes steps to create/use a Supabase Secret API key, place it in the mounted secret file matching password_file, reload/restart Prometheus via Compose or lifecycle reload, and verify via Prometheus targets showing the supabase job UP." + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -5372,41 +4185,12 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"project metrics prometheus endpoint observability\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" - } - ], - "resultChars": 29057 - } - ] + "calls": [] }, - "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", - "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5/deploy-database-001-prometheus-metrics.json" + "sourcePath": "claude-code-sonnet-5/build-dataapi-001-relational-report.json" }, { "experiment": "claude-code-sonnet-5", @@ -5417,35 +4201,44 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "deploy-functions-001-edge-function-secrets", - "stage": "deploy", + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", "product": [ - "edge-functions" + "data-api", + "database" ], "topic": [ - "security" + "sdk" ], "suite": "benchmark", "interface": "cli", + "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "WEATHER_API_KEY is set as a Function secret on the project", - "passed": true + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" }, { - "name": "the weather function is deployed to the project", + "name": "alerts match the database (below threshold, sorted)", "passed": true, - "notes": "status ACTIVE" + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" }, { - "name": "the weather function reads WEATHER_API_KEY from the environment", + "name": "tables stay locked down (publishable key reads nothing)", "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + "notes": "publishable read errored: permission denied for table inventory" }, { - "name": "WEATHER_API_KEY value is not committed to the repo", - "passed": true + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/restock.mjs" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -5453,74 +4246,75 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [ - "supabase" - ] + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Functions environment variables secrets Deno.env deploy\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", - "title": "Inspecting edge function environment variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", - "title": "Testing and linting" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - } - ], - "resultChars": 38791 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"management API invoke edge function endpoint\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/architecture", - "title": "Edge Functions Architecture" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-functions", - "title": "Manage Supabase Edge functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - } - ], - "resultChars": 66572 - } + "calls": [] + }, + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5/build-dataapi-002-restock-alert-report.json" + }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-database-001-migrate-postgres-to-supabase", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "all 3 tables exist (teams, members, tasks)", + "passed": true + }, + { + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true + }, + { + "name": "foreign key constraints survived the restore", + "passed": true + }, + { + "name": "tasks_team_status_idx index survived the restore", + "passed": true + }, + { + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" ] }, - "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", - "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", + "docs": { + "calls": [] + }, + "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", + "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5/deploy-functions-001-edge-function-secrets.json" + "sourcePath": "claude-code-sonnet-5/build-database-001-migrate-postgres-to-supabase.json" }, { "experiment": "claude-code-sonnet-5", @@ -5531,35 +4325,46 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", + "eval": "build-functions-004-service-role-bypass", + "stage": "build", "product": [ - "database", + "edge-functions", "auth", - "storage" + "database" ], "topic": [ - "self-hosting" + "rls", + "security", + "sdk" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", - "passed": true + "name": "rejects missing auth", + "passed": true, + "notes": "status=401" }, { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", - "passed": true + "name": "user A reads own note", + "passed": true, + "notes": "status=200" }, { - "name": "secrets rotated off the shipped defaults", - "passed": true + "name": "reads only with the caller's JWT", + "passed": true, + "notes": "bearer_tokens=2, all_match=true" }, { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", - "passed": true + "name": "user A cannot force-read user B note", + "passed": true, + "notes": "status=200" + }, + { + "name": "user B cannot force-read user A note", + "passed": true, + "notes": "status=200" } ], "skills": { @@ -5574,10 +4379,10 @@ "docs": { "calls": [] }, - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5/deploy-self-hosting-001-docker-compose.json" + "sourcePath": "claude-code-sonnet-5/build-functions-004-service-role-bypass.json" }, { "experiment": "claude-code-sonnet-5", @@ -5588,49 +4393,67 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", + "eval": "build-functions-005-dual-auth-user-secret", + "stage": "build", "product": [ - "auth" + "edge-functions", + "auth", + "database" ], "topic": [ - "security", - "sdk" + "sdk", + "rls", + "security" ], "suite": "benchmark", - "interface": "mcp", - "passed": false, + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, "checks": [ { - "name": "victim session active before delete-account", - "passed": true + "name": "seed rows present", + "passed": true, + "notes": "found 2/2 seeded rows" }, { - "name": "delete_account flow ran for the victim", - "passed": true + "name": "rejects request with no credentials", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { - "name": "delete-account revokes the user's sessions", + "name": "user with JWT reads only their own rows", "passed": true, - "notes": "sessions left: 0" + "notes": "status 200: {\"data\":[{\"user_id\":\"4b5af664-544f-484c-8c95-9569c5445834\",\"metric\":\"steps_a_mssz4hex\",\"value\":111}]}" }, { - "name": "deleted user's refresh token is rejected", - "passed": true + "name": "user cannot read another user's rows by passing user_id", + "passed": true, + "notes": "status 200: {\"data\":[{\"user_id\":\"4b5af664-544f-484c-8c95-9569c5445834\",\"metric\":\"steps_a_mssz4hex\",\"value\":111}]}" }, { - "name": "deleted user cannot sign back in", - "passed": false, - "notes": "deleted account can still sign in" + "name": "service key bypasses RLS to read the target user's rows", + "passed": true, + "notes": "status 200: {\"data\":[{\"user_id\":\"d426d621-c66e-4559-8a25-3e24c7fe273c\",\"metric\":\"steps_b_mssz4hex\",\"value\":222}]}" }, { - "name": "other users keep their sessions and access", - "passed": true + "name": "non-service key is not granted service access", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { - "name": "diagnosed and explained session revocation", + "name": "rejects an unverified (forged) user token", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "a user token in the apikey slot is not treated as the service key", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "implementation uses @supabase/server", "passed": true, - "judgeNotes": "The answer identifies the soft-delete-only bug, implements real revocation by deleting auth.sessions and banning the auth user, adds RLS checks to close the data-access stale-JWT window, and consistently explains that stateless JWTs remain locally valid until expiry. It also correctly distinguishes frontend publishable/anon keys from server-only secret/service_role keys and notes secret bypasses RLS." + "notes": "imports @supabase/server / withSupabase" } ], "skills": { @@ -5643,12 +4466,125 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Edge Functions environment variables SUPABASE_SERVICE_ROLE_KEY SUPABASE_ANON_KEY apikey header\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/storage-caching", + "title": "Integrating with Supabase Storage" + } + ], + "resultChars": 36932 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"new API keys publishable secret key edge functions\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", + "title": "Getting Started with Edge Functions (Dashboard)" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + } + ], + "resultChars": 65708 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Securing Edge Functions auth modes withSupabase user secret publishable multiple auth\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + }, + { + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + } + ], + "resultChars": 43314 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"@supabase/server SDK ctx.supabase ctx.supabaseAdmin auth modes array withSupabase\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/roles", + "title": "Postgres Roles" + }, + { + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", + "title": "Which package to use" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + } + ], + "resultChars": 48802 + } + ] }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", + "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", "attempts": 2, - "sourcePath": "claude-code-sonnet-5/investigate-auth-001-deleted-user-access.json" + "sourcePath": "claude-code-sonnet-5/build-functions-005-dual-auth-user-secret.json" }, { "experiment": "claude-code-sonnet-5", @@ -5659,13 +4595,14 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "investigate-realtime-001-subscribed-no-events", - "stage": "investigate", + "eval": "build-storage-001-private-bucket-access", + "stage": "build", "product": [ - "realtime", + "storage", "database" ], "topic": [ + "rls", "sdk" ], "suite": "benchmark", @@ -5673,30 +4610,42 @@ "passed": true, "checks": [ { - "name": "orders table added to supabase_realtime publication", + "name": "bucket user-files exists", "passed": true }, { - "name": "courier_locations still in supabase_realtime publication", + "name": "bucket user-files is private", "passed": true }, { - "name": "publication still publishes INSERT events", + "name": "RLS still enabled on storage.objects", "passed": true }, { - "name": "RLS still enabled on orders", + "name": "user A lists only own files", + "passed": true, + "notes": "saw: 01a0005d-8f41-776c-8ada-a927b206e62d/receipt-alpha.pdf, 01a0005d-8f41-776c-8ada-a927b206e62d/receipt-beta.pdf" + }, + { + "name": "user B cannot read user A files", "passed": true }, { - "name": "staff can still read orders through RLS", - "passed": true, - "notes": "authenticated sees 2 of 2 orders" + "name": "anon reads no files", + "passed": true }, { - "name": "diagnosed missing publication membership", + "name": "user A can upload into own folder", + "passed": true + }, + { + "name": "user B cannot upload into user A folder", + "passed": true + }, + { + "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "The assistant correctly identified the root cause as orders missing from the supabase_realtime publication despite SUBSCRIBED status, added only public.orders to the existing publication, preserved courier_locations/RLS/policies, and did not blame or alter client code, RLS, grants, or networking." + "judgeNotes": "Meets requirements: private user-files bucket, authenticated owner-scoped SELECT and INSERT policies with WITH CHECK using foldername/auth.uid, RLS not disabled, and supabase-js createSignedUrl with expiry. No disallowed public bucket/getPublicUrl/service-role usage." } ], "skills": { @@ -5709,12 +4658,65 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"storage private bucket RLS policy owner folder\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + }, + { + "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", + "title": "Storage Helper Functions" + }, + { + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" + }, + { + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" + } + ], + "resultChars": 23289 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"createSignedUrl storage signed url expiring share\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" + }, + { + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" + }, + { + "url": "https://supabase.com/docs/guides/storage/serving/downloads", + "title": "Serving assets from Storage" + }, + { + "url": "https://supabase.com/docs/reference/dart/file-buckets-createsignedurl" + }, + { + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" + } + ], + "resultChars": 7638 + } + ] }, - "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", - "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", + "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", + "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5/investigate-realtime-001-subscribed-no-events.json" + "sourcePath": "claude-code-sonnet-5/build-storage-001-private-bucket-access.json" }, { "experiment": "claude-code-sonnet-5", @@ -5725,32 +4727,33 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "investigate-reliability-003-edge-function-5xx-correlation", - "stage": "investigate", + "eval": "build-tests-001-rls-tenant-isolation", + "stage": "build", "product": [ - "edge-functions" + "database" ], "topic": [ - "observability" + "tests", + "rls" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", "passed": true, "checks": [ { - "name": "identified image-transform and the recurring 503 pattern", + "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "judgeNotes": "Identified image-transform as the main affected function and described the recurring HTTP 503 pattern across the 2026-04-28 morning window, including the eight gateway failures from about 07:00Z to 12:00Z. Also correctly distinguished older billing-webhook 503s as unrelated." + "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" }, { - "name": "attributed recurring 503s to gateway/platform layer, not function code", + "name": "pgTAP isolation tests ran and pass", "passed": true, - "judgeNotes": "Attributes image-transform 503s to upstream gateway/platform rejection rather than handler code, grounded in missing edge-function runtime logs for 503s while 200s appear, and distinguishes avatar-upload's runtime 500 as a separate function-level error." + "notes": "8 passed, 3 failed" }, { - "name": "recommended a concrete next step", + "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The assistant provided multiple concrete actionable next steps, including instrumenting memory/input sizes, checking for a recurring scheduler/batch job, adding retry/backoff, investigating the separate avatar-upload stack trace, and considering architectural changes for heavy image processing." + "judgeNotes": "The agent correctly identifies `posts` as having a broken tenant isolation SELECT policy, specifically allowing authenticated members of any org to read posts from other orgs, and grounds this in pgTAP failures. It also correctly states `notes` is isolated and treats test results as authoritative. Extra findings about `memberships` do not undermine the required conclusion." } ], "skills": { @@ -5759,16 +4762,17 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { "calls": [] }, - "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", - "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", + "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", + "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5/investigate-reliability-003-edge-function-5xx-correlation.json" + "sourcePath": "claude-code-sonnet-5/build-tests-001-rls-tenant-isolation.json" }, { "experiment": "claude-code-sonnet-5", @@ -5779,49 +4783,50 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "resolve-dataapi-001-empty-results", - "stage": "resolve", + "eval": "build-vectors-001-rag-with-permissions", + "stage": "build", "product": [ - "data-api", "database", - "auth" + "vectors" ], "topic": [ - "rls", - "sdk" + "sql", + "rls" ], "suite": "benchmark", "interface": "mcp", "passed": true, "checks": [ { - "name": "RLS still enabled on bookmarks", - "passed": true + "name": "document_sections.embedding is vector(384)", + "passed": true, + "notes": "vector(384)" }, { - "name": "user A reads own bookmarks", - "passed": true + "name": "HNSW index on the embedding column", + "passed": true, + "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" }, { - "name": "user B cannot read user A bookmarks", - "passed": true + "name": "index operator class matches the search operator", + "passed": true, + "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" }, { - "name": "anon reads no bookmarks", + "name": "user A search returns only own sections, best match first", "passed": true }, { - "name": "user A can save a new bookmark", + "name": "user B search returns only own sections, best match first", "passed": true }, { - "name": "user B cannot insert a bookmark as user A", + "name": "user A reads only own sections through the API", "passed": true }, { - "name": "diagnosed RLS and added owner-scoped policies", - "passed": true, - "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies causing deny-all Data API results, kept RLS enabled, created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for insert, and verified behavior." + "name": "user A reads only own documents through the API", + "passed": true } ], "skills": { @@ -5830,16 +4835,34 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"match_document_sections gte-small semantic search edge function\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", + "title": "Semantic Search" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" + } + ], + "resultChars": 19592 + } + ] }, - "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", - "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", + "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", + "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5/resolve-dataapi-001-empty-results.json" + "sourcePath": "claude-code-sonnet-5/build-vectors-001-rag-with-permissions.json" }, { "experiment": "claude-code-sonnet-5", @@ -5850,42 +4873,30 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "resolve-database-001-migration-history-mismatch", - "stage": "resolve", + "eval": "deploy-database-001-prometheus-metrics", + "stage": "deploy", "product": [ "database" ], "topic": [ - "migrations" + "observability" ], "suite": "benchmark", - "interface": "cli", "passed": true, "checks": [ { - "name": "the avatar_url column is applied on the hosted profiles table", - "passed": true - }, - { - "name": "migration 20240220000000 is recorded in the remote history", - "passed": true - }, - { - "name": "remote migration history matches local migration files", - "passed": true - }, - { - "name": "local migrations are a valid reconciled sequence", + "name": "preserved existing app scrape job", "passed": true }, { - "name": "production profile data is intact (not reset)", - "passed": true + "name": "configured the Supabase Metrics API scrape correctly", + "passed": true, + "judgeNotes": "Prometheus config preserves the app scrape and adds a Supabase job using HTTPS, the required /customer/v1/privileged/metrics path, a .supabase.co target, and HTTP Basic Auth with password_file. docker-compose mounts ./secrets to the matching /etc/prometheus/secrets path." }, { - "name": "the avatar migration and history reconciliation were done via the Supabase CLI", + "name": "documented live deployment and verification steps", "passed": true, - "judgeNotes": "Applied avatar_url via `supabase db push` in #11, which showed `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` Reconciled the orphan bio migration by adding local file `20240115000000_add_profile_bio.sql` in #9, after which `supabase migration list` matched local/remote and `db push` proceeded. No disallowed workaround or direct mutation observed; psql usage was read-only inspection." + "judgeNotes": "README includes creating a Supabase Secret API key, writing it to the expected Prometheus password_file path, a Compose restart command for Prometheus, and concrete verification via Prometheus targets plus Grafana dashboard." } ], "skills": { @@ -5898,12 +4909,53 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Prometheus metrics endpoint project\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/api/v1-scrape-project-metrics", + "title": "Scrape a project's metrics" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", + "title": "Metrics API" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" + } + ], + "resultChars": 20571 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"prometheus password_file basic_auth reload\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/passkeys", + "title": "Passkey authentication" + } + ], + "resultChars": 11900 + } + ] }, - "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", - "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", + "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", + "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5/resolve-database-001-migration-history-mismatch.json" + "sourcePath": "claude-code-sonnet-5/deploy-database-001-prometheus-metrics.json" }, { "experiment": "claude-code-sonnet-5", @@ -5914,38 +4966,34 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "resolve-performance-001-slow-query-cpu-spike", - "stage": "resolve", + "eval": "deploy-functions-001-edge-function-secrets", + "stage": "deploy", "product": [ - "database" + "edge-functions" ], "topic": [ - "observability", - "sql" + "security" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", "passed": true, "checks": [ { - "name": "inspected pg_stat_statements for query performance", - "passed": true - }, - { - "name": "ran EXPLAIN on the expensive query", + "name": "WEATHER_API_KEY is set as a Function secret on the project", "passed": true }, { - "name": "created index covering user_id and created_at", - "passed": true + "name": "the weather function is deployed to the project", + "passed": true, + "notes": "status ACTIVE" }, { - "name": "query plan uses an index and avoids sequential scan", + "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get('WEATHER_API_KEY')." }, { - "name": "inserts still work", + "name": "WEATHER_API_KEY value is not committed to the repo", "passed": true } ], @@ -5955,17 +5003,101 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge function environment variables secrets Deno.env\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/functions/unit-test", + "title": "Testing your Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", + "title": "Inspecting edge function environment variables" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" + } + ], + "resultChars": 41263 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"management API invoke edge function endpoint POST /v1/projects/{ref}/functions\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/api/v1-create-a-function", + "title": "Create a function" + }, + { + "url": "https://supabase.com/docs/reference/api/v1-deploy-a-function", + "title": "Deploy a function" + }, + { + "url": "https://supabase.com/docs/reference/api/v1-list-all-functions", + "title": "List all functions" + }, + { + "url": "https://supabase.com/docs/reference/api/v1-bulk-update-functions", + "title": "Bulk update functions" + }, + { + "url": "https://supabase.com/docs/reference/api/v1-update-a-function", + "title": "Update a function" + } + ], + "resultChars": 3631 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"get function body source\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", + "title": "Getting Started with Edge Functions (Dashboard)" + }, + { + "url": "https://supabase.com/docs/reference/api/v1-get-a-function-body", + "title": "Retrieve a function body" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/log-drains", + "title": "Log Drains" + } + ], + "resultChars": 50872 + } + ] }, - "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", - "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", + "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", + "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5/resolve-performance-001-slow-query-cpu-spike.json" + "sourcePath": "claude-code-sonnet-5/deploy-functions-001-edge-function-secrets.json" }, { "experiment": "claude-code-sonnet-5", @@ -5976,54 +5108,34 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "resolve-security-002-rls-cross-tenant-leak", - "stage": "resolve", + "eval": "deploy-self-hosting-001-docker-compose", + "stage": "deploy", "product": [ "database", - "auth" + "auth", + "storage" ], "topic": [ - "rls", - "security" + "self-hosting" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", "passed": true, "checks": [ { - "name": "RLS enabled on notes", - "passed": true - }, - { - "name": "tenant A sees only org A notes", - "passed": true - }, - { - "name": "tenant B cannot read org A notes", - "passed": true - }, - { - "name": "tenant A author can update own note", - "passed": true - }, - { - "name": "tenant B cannot update org A note", - "passed": true - }, - { - "name": "tenant B author can delete own note", + "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", "passed": true }, { - "name": "tenant B cannot delete org A note", + "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", "passed": true }, { - "name": "tenant A can insert note in own org", + "name": "secrets rotated off the shipped defaults", "passed": true }, { - "name": "tenant B cannot insert into org A", + "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", "passed": true } ], @@ -6039,434 +5151,471 @@ "docs": { "calls": [] }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", + "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5/resolve-security-002-rls-cross-tenant-leak.json" + "sourcePath": "claude-code-sonnet-5/deploy-self-hosting-001-docker-compose.json" }, { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "build-auth-001-email-password-flow", - "stage": "build", + "eval": "investigate-auth-001-deleted-user-access", + "stage": "investigate", "product": [ - "auth", - "database" + "auth" ], "topic": [ - "sdk", - "rls" + "security", + "sdk" ], "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", + "interface": "mcp", "passed": true, "checks": [ { - "name": "auth module loads and the driver completes", - "passed": true, - "notes": "driver produced a result" - }, - { - "name": "signUp creates the account and returns its user id", - "passed": true, - "notes": "db user 863566c0-fe1b-4264-9906-53f26c545be8, signUp returned {\"userId\":\"863566c0-fe1b-4264-9906-53f26c545be8\"}" + "name": "victim session active before delete-account", + "passed": true }, { - "name": "signup metadata reaches the profile (display name)", - "passed": true, - "notes": "profiles.display_name = \"Alex Doe\"" + "name": "delete_account flow ran for the victim", + "passed": true }, { - "name": "wrong password is rejected gracefully (no throw, no session)", + "name": "delete-account revokes the user's sessions", "passed": true, - "notes": "{\"error\":\"Invalid login credentials\"}" + "notes": "sessions left: 0" }, { - "name": "signIn with the right password returns the user id", - "passed": true, - "notes": "{\"userId\":\"863566c0-fe1b-4264-9906-53f26c545be8\"}" + "name": "deleted user's refresh token is rejected", + "passed": true }, { - "name": "getMyProfile returns the signed-in user's profile", - "passed": true, - "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + "name": "deleted user cannot sign back in", + "passed": true }, { - "name": "app code does not use the secret / service-role key", - "passed": true, - "notes": "no secret-key references found" + "name": "other users keep their sessions and access", + "passed": true }, { - "name": "implementation uses @supabase/supabase-js", + "name": "diagnosed and explained session revocation", "passed": true, - "notes": "imports found in: app/src/auth.mjs" + "judgeNotes": "Diagnoses soft delete/no auth revocation, implements hard delete of auth.users with session/refresh-token cascade and FK cleanup, explains remaining stateless JWT window consistently with its fix, and correctly distinguishes publishable frontend/RLS-enforced keys from secret server-only/RLS-bypassing keys." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] }, "docs": { "calls": [] }, - "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", - "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-auth-001-email-password-flow.json" + "sourcePath": "claude-code-sonnet-5/investigate-auth-001-deleted-user-access.json" }, { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "build-cli-001-bootstrap-app", - "stage": "build", + "eval": "investigate-realtime-001-subscribed-no-events", + "stage": "investigate", "product": [ - "database", - "data-api" + "realtime", + "database" ], "topic": [ - "migrations", - "rls" + "sdk" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "supabase project initialised (supabase/config.toml exists)", + "name": "orders table added to supabase_realtime publication", "passed": true }, { - "name": "todos table is created by a migration file", + "name": "courier_locations still in supabase_realtime publication", "passed": true }, { - "name": "todos table exists with at least 2 seeded rows", - "passed": true, - "notes": "found 2 rows" - }, - { - "name": "row level security is enabled on todos", + "name": "publication still publishes INSERT events", "passed": true }, { - "name": "a SELECT policy targets the authenticated role", + "name": "RLS still enabled on orders", "passed": true }, { - "name": "REST API returns no todos to anonymous requests", + "name": "staff can still read orders through RLS", "passed": true, - "notes": "0 rows" + "notes": "authenticated sees 2 of 2 orders" }, { - "name": "REST API returns the todos to authenticated requests", + "name": "diagnosed missing publication membership", "passed": true, - "notes": "2 rows" + "judgeNotes": "Identified missing orders table in supabase_realtime publication as root cause, added only public.orders to the existing publication, and did not alter RLS/policies or disrupt courier_locations." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] }, "docs": { "calls": [] }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", + "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", + "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-cli-001-bootstrap-app.json" + "sourcePath": "claude-code-sonnet-5/investigate-realtime-001-subscribed-no-events.json" }, { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "build-cli-002-declarative-schema", - "stage": "build", + "eval": "investigate-reliability-003-edge-function-5xx-correlation", + "stage": "investigate", "product": [ - "database" + "edge-functions" ], "topic": [ - "declarative-schema", - "migrations" + "observability" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "supabase db diff used to generate the migration", - "passed": true - }, - { - "name": "schema file updated to include description column", - "passed": true + "name": "identified image-transform and the recurring 503 pattern", + "passed": true, + "judgeNotes": "Identified image-transform as the affected function and described the recurring 503 pattern across the morning of 2026-04-28, covering all eight gateway failures from 07:00Z to 12:00Z. Also distinguished old billing-webhook 503s as unrelated." }, { - "name": "a new migration was generated for the change", - "passed": true + "name": "attributed recurring 503s to gateway/platform layer, not function code", + "passed": true, + "judgeNotes": "The assistant clearly attributes the recurring image-transform 503s to the API gateway/platform layer before the function runtime, not to function application code. This is grounded in valid observations: the 503s appear without corresponding edge-function execution log rows while nearby 200s do, and the avatar-upload 500 is distinguished as a separate function-level runtime error." }, { - "name": "description column exists in the live database", - "passed": true + "name": "recommended a concrete next step", + "passed": true, + "judgeNotes": "The assistant recommended multiple concrete actionable next steps, including checking bundle/dependency size, implementing retries, reducing cold-start time or keeping the function warm, and separately investigating the avatar-upload error with structured logging." } ], "skills": { - "available": [], + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], "loaded": [] }, "docs": { "calls": [] }, - "prompt": "Add a description text column to the `products` table in my local Supabase stack", - "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", + "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", + "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-cli-002-declarative-schema.json" + "sourcePath": "claude-code-sonnet-5/investigate-reliability-003-edge-function-5xx-correlation.json" }, { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "build-cli-003-pg-cron-queue-workflow", - "stage": "build", + "eval": "resolve-dataapi-001-empty-results", + "stage": "resolve", "product": [ + "data-api", "database", - "edge-functions", - "cron", - "queues" + "auth" ], "topic": [ - "sql", + "rls", "sdk" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" + "name": "RLS still enabled on bookmarks", + "passed": true }, { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 0 -> 1" + "name": "user A reads own bookmarks", + "passed": true }, { - "name": "process-tasks function drains the queue", + "name": "user B cannot read user A bookmarks", + "passed": true + }, + { + "name": "anon reads no bookmarks", + "passed": true + }, + { + "name": "user A can save a new bookmark", + "passed": true + }, + { + "name": "user B cannot insert a bookmark as user A", + "passed": true + }, + { + "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "notes": "function removed the seeded message (id 4) from the queue" + "judgeNotes": "Diagnosed RLS enabled with no policies causing deny-all Data API results, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid(), with INSERT enforced via WITH CHECK." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] }, "docs": { "calls": [] }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", + "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-cli-003-pg-cron-queue-workflow.json" + "sourcePath": "claude-code-sonnet-5/resolve-dataapi-001-empty-results.json" }, { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "build-dataapi-001-relational-report", - "stage": "build", + "eval": "resolve-database-001-migration-history-mismatch", + "stage": "resolve", "product": [ - "data-api", "database" ], "topic": [ - "sdk" + "migrations" ], "suite": "benchmark", "interface": "cli", - "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" + "name": "the avatar_url column is applied on the hosted profiles table", + "passed": true }, { - "name": "report numbers match the database (per customer, sorted)", - "passed": true, - "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + "name": "migration 20240220000000 is recorded in the remote history", + "passed": true }, { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table customers" + "name": "remote migration history matches local migration files", + "passed": true }, { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/report.mjs" + "name": "local migrations are a valid reconciled sequence", + "passed": true }, { - "name": "report queries via the Data API, not raw SQL", + "name": "production profile data is intact (not reset)", + "passed": true + }, + { + "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "notes": "no psql / raw Postgres driver usage found" + "judgeNotes": "Avatar_url was applied through `supabase db push` in action #17, with output `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration was reconciled by adding the missing local file `20240115000000_add_profile_bio.sql` in action #15, after which `supabase migration list` showed local and remote aligned in action #16. The psql commands were read-only inspection only; no prohibited direct mutation or workaround was used." } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] }, "docs": { "calls": [] }, - "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", - "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 2, - "sourcePath": "claude-code-sonnet-5-no-skills/build-dataapi-001-relational-report.json" + "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", + "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5/resolve-database-001-migration-history-mismatch.json" }, { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "build-dataapi-002-restock-alert-report", - "stage": "build", + "eval": "resolve-performance-001-slow-query-cpu-spike", + "stage": "resolve", "product": [ - "data-api", "database" ], "topic": [ - "sdk" + "observability", + "sql" ], "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, + "interface": "mcp", + "passed": true, "checks": [ { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" + "name": "inspected pg_stat_statements for query performance", + "passed": true }, { - "name": "alerts match the database (below threshold, sorted)", - "passed": true, - "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + "name": "ran EXPLAIN on the expensive query", + "passed": true }, { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table inventory" + "name": "created index covering user_id and created_at", + "passed": true }, { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + "name": "query plan uses an index and avoids sequential scan", + "passed": true, + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" + "name": "inserts still work", + "passed": true } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [] }, - "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", - "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", - "attempts": 2, - "sourcePath": "claude-code-sonnet-5-no-skills/build-dataapi-002-restock-alert-report.json" + "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", + "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5/resolve-performance-001-slow-query-cpu-spike.json" }, { - "experiment": "claude-code-sonnet-5-no-skills", - "experimentSuite": "no-skills", + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "build-database-001-migrate-postgres-to-supabase", - "stage": "build", + "eval": "resolve-security-002-rls-cross-tenant-leak", + "stage": "resolve", "product": [ - "database" + "database", + "auth" ], "topic": [ - "migrations" + "rls", + "security" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "all 3 tables exist (teams, members, tasks)", + "name": "RLS enabled on notes", + "passed": true + }, + { + "name": "tenant A sees only org A notes", + "passed": true + }, + { + "name": "tenant B cannot read org A notes", "passed": true }, { - "name": "row counts match (teams=5, members=10, tasks=13)", + "name": "tenant A author can update own note", "passed": true }, { - "name": "foreign key constraints survived the restore", + "name": "tenant B cannot update org A note", "passed": true }, { - "name": "tasks_team_status_idx index survived the restore", + "name": "tenant B author can delete own note", "passed": true }, { - "name": "sequences synced (next insert won't conflict with existing IDs)", + "name": "tenant B cannot delete org A note", + "passed": true + }, + { + "name": "tenant A can insert note in own org", + "passed": true + }, + { + "name": "tenant B cannot insert into org A", "passed": true } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] }, "docs": { "calls": [] }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", + "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-database-001-migrate-postgres-to-supabase.json" + "sourcePath": "claude-code-sonnet-5/resolve-security-002-rls-cross-tenant-leak.json" }, { "experiment": "claude-code-sonnet-5-no-skills", @@ -6477,46 +5626,60 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "build-functions-004-service-role-bypass", + "eval": "build-auth-001-email-password-flow", "stage": "build", "product": [ - "edge-functions", "auth", "database" ], "topic": [ - "rls", - "security", - "sdk" + "sdk", + "rls" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", + "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "rejects missing auth", + "name": "auth module loads and the driver completes", "passed": true, - "notes": "status=401" + "notes": "driver produced a result" }, { - "name": "user A reads own note", + "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "status=200" + "notes": "db user f3102f30-9597-4b1f-9763-a6b58405b648, signUp returned {\"userId\":\"f3102f30-9597-4b1f-9763-a6b58405b648\"}" }, { - "name": "reads only with the caller's JWT", + "name": "signup metadata reaches the profile (display name)", "passed": true, - "notes": "bearer_tokens=2, all_match=true" + "notes": "profiles.display_name = \"Alex Doe\"" }, { - "name": "user A cannot force-read user B note", + "name": "wrong password is rejected gracefully (no throw, no session)", "passed": true, - "notes": "status=200" + "notes": "{\"error\":\"Invalid login credentials\"}" }, { - "name": "user B cannot force-read user A note", + "name": "signIn with the right password returns the user id", "passed": true, - "notes": "status=200" + "notes": "{\"userId\":\"f3102f30-9597-4b1f-9763-a6b58405b648\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" } ], "skills": { @@ -6526,10 +5689,10 @@ "docs": { "calls": [] }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 2, - "sourcePath": "claude-code-sonnet-5-no-skills/build-functions-004-service-role-bypass.json" + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/build-auth-001-email-password-flow.json" }, { "experiment": "claude-code-sonnet-5-no-skills", @@ -6540,67 +5703,50 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "build-functions-005-dual-auth-user-secret", + "eval": "build-cli-001-bootstrap-app", "stage": "build", "product": [ - "edge-functions", - "auth", - "database" + "database", + "data-api" ], "topic": [ - "sdk", - "rls", - "security" + "migrations", + "rls" ], "suite": "benchmark", "interface": "cli", - "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"6f5b2171-8fac-4f4d-a612-8f5898da836f\",\"metric\":\"steps_a_msj0y6nu\",\"value\":111}]}" + "name": "supabase project initialised (supabase/config.toml exists)", + "passed": true }, { - "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"6f5b2171-8fac-4f4d-a612-8f5898da836f\",\"metric\":\"steps_a_msj0y6nu\",\"value\":111}]}" + "name": "todos table is created by a migration file", + "passed": true }, { - "name": "service key bypasses RLS to read the target user's rows", + "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"36482d33-6a70-43e3-bd27-459a7428cf03\",\"metric\":\"steps_b_msj0y6nu\",\"value\":222}]}" + "notes": "found 2 rows" }, { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "name": "row level security is enabled on todos", + "passed": true }, { - "name": "rejects an unverified (forged) user token", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "name": "a SELECT policy targets the authenticated role", + "passed": true }, { - "name": "a user token in the apikey slot is not treated as the service key", + "name": "REST API returns no todos to anonymous requests", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "0 rows" }, { - "name": "implementation uses @supabase/server", + "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "imports @supabase/server / withSupabase" + "notes": "2 rows" } ], "skills": { @@ -6608,95 +5754,12 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md 2>&1 | head -100", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 5516 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Functions environment variables SUPABASE_SERVICE_ROLE_KEY SUPABASE_URL default secrets\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - } - ], - "resultChars": 41771 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"new API keys publishable secret key edge functions env var migration from anon service_role\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", - "title": "Backup and Restore using the CLI" - } - ], - "resultChars": 95976 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Authorization headers verify_jwt combining auth modes user secret apikey Edge Functions\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - } - ], - "resultChars": 24799 - } - ] + "calls": [] }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", + "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", + "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-functions-005-dual-auth-user-secret.json" + "sourcePath": "claude-code-sonnet-5-no-skills/build-cli-001-bootstrap-app.json" }, { "experiment": "claude-code-sonnet-5-no-skills", @@ -6707,57 +5770,34 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "build-storage-001-private-bucket-access", - "stage": "build", - "product": [ - "storage", - "database" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", - "passed": true - }, - { - "name": "RLS still enabled on storage.objects", - "passed": true - }, - { - "name": "user A lists only own files", - "passed": true, - "notes": "saw: 019fdc8c-98ed-736c-a6e0-4477d1da6052/receipt-alpha.pdf, 019fdc8c-98ed-736c-a6e0-4477d1da6052/receipt-beta.pdf" - }, + "eval": "build-cli-002-declarative-schema", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "declarative-schema", + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ { - "name": "user B cannot read user A files", + "name": "supabase db diff used to generate the migration", "passed": true }, { - "name": "anon reads no files", + "name": "schema file updated to include description column", "passed": true }, { - "name": "user A can upload into own folder", + "name": "a new migration was generated for the change", "passed": true }, { - "name": "user B cannot upload into user A folder", + "name": "description column exists in the live database", "passed": true - }, - { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "Meets requirements: private user-files bucket, authenticated owner-scoped SELECT and INSERT policies on storage.objects with WITH CHECK for uploads, no RLS disabling or public access, and supabase-js createSignedUrl with expiry for temporary sharing." } ], "skills": { @@ -6767,10 +5807,10 @@ "docs": { "calls": [] }, - "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", - "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", + "prompt": "Add a description text column to the `products` table in my local Supabase stack", + "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-storage-001-private-bucket-access.json" + "sourcePath": "claude-code-sonnet-5-no-skills/build-cli-002-declarative-schema.json" }, { "experiment": "claude-code-sonnet-5-no-skills", @@ -6781,33 +5821,36 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "build-tests-001-rls-tenant-isolation", + "eval": "build-cli-003-pg-cron-queue-workflow", "stage": "build", "product": [ - "database" + "database", + "edge-functions", + "cron", + "queues" ], "topic": [ - "tests", - "rls" + "sql", + "sdk" ], "suite": "benchmark", "interface": "cli", "passed": true, "checks": [ { - "name": "pgTAP test file(s) written under supabase/tests/", + "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", "passed": true, - "notes": "1 file(s): supabase/tests/database/tenant_isolation_test.sql" + "notes": "schedule='* * * * *', active=true" }, { - "name": "pgTAP isolation tests ran and pass", + "name": "cron command enqueues to the 'tasks' queue", "passed": true, - "notes": "10 passed, 5 failed" + "notes": "queue depth 0 -> 1" }, { - "name": "agent correctly identifies the posts isolation bug from test results", + "name": "process-tasks function drains the queue", "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as having the tenant isolation flaw, explains the missing `org_id` predicate, and grounds the conclusion in the pgTAP failures showing cross-org post visibility. It does not blame `notes` and treats the test results as authoritative." + "notes": "function removed the seeded message (id 3) from the queue" } ], "skills": { @@ -6815,12 +5858,121 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"pg_cron schedule edge function invoke http\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + }, + { + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart", + "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" + } + ], + "resultChars": 58538 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"pgmq queues cron worker example send read delete\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" + }, + { + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" + } + ], + "resultChars": 34511 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"cron.schedule syntax example every minute\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/cron/quickstart", + "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_cron", + "title": "pg_cron: Schedule Recurring Jobs with Cron Syntax in Postgres" + } + ], + "resultChars": 23235 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"trigger edge function from database webhooks local development url host\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/webhooks", + "title": "Database Webhooks" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-hooks", + "title": "Auth Hooks" + }, + { + "url": "https://supabase.com/docs/guides/realtime/broadcast", + "title": "Broadcast" + } + ], + "resultChars": 60557 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge function example Deno.serve createClient service role key queue worker\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", + "title": "Testing and linting" + } + ], + "resultChars": 30748 + } + ] }, - "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", - "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-tests-001-rls-tenant-isolation.json" + "sourcePath": "claude-code-sonnet-5-no-skills/build-cli-003-pg-cron-queue-workflow.json" }, { "experiment": "claude-code-sonnet-5-no-skills", @@ -6831,50 +5983,44 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "build-vectors-001-rag-with-permissions", + "eval": "build-dataapi-001-relational-report", "stage": "build", "product": [ - "database", - "vectors" + "data-api", + "database" ], "topic": [ - "sql", - "rls" + "sdk" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", + "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "document_sections.embedding is vector(384)", + "name": "report runs and prints JSON", "passed": true, - "notes": "vector(384)" + "notes": "exit 0" }, { - "name": "HNSW index on the embedding column", + "name": "report numbers match the database (per customer, sorted)", "passed": true, - "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" }, { - "name": "index operator class matches the search operator", + "name": "tables stay locked down (publishable key reads nothing)", "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" - }, - { - "name": "user A search returns only own sections, best match first", - "passed": true - }, - { - "name": "user B search returns only own sections, best match first", - "passed": true + "notes": "publishable read errored: permission denied for table customers" }, { - "name": "user A reads only own sections through the API", - "passed": true + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/report.mjs" }, { - "name": "user A reads only own documents through the API", - "passed": true + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -6884,10 +6030,10 @@ "docs": { "calls": [] }, - "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", - "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-vectors-001-rag-with-permissions.json" + "sourcePath": "claude-code-sonnet-5-no-skills/build-dataapi-001-relational-report.json" }, { "experiment": "claude-code-sonnet-5-no-skills", @@ -6898,72 +6044,57 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "deploy-database-001-prometheus-metrics", - "stage": "deploy", + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", "product": [ + "data-api", "database" ], "topic": [ - "observability" + "sdk" ], "suite": "benchmark", - "passed": true, + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, "checks": [ { - "name": "preserved existing app scrape job", - "passed": true - }, - { - "name": "configured the Supabase Metrics API scrape correctly", + "name": "report runs and prints JSON", "passed": true, - "judgeNotes": "Meets all requirements: Supabase Metrics API scrape uses HTTPS, correct metrics path, Basic Auth with password_file, valid supabase.co project target, app scrape is preserved, and docker-compose mounts the secrets directory containing the password file." + "notes": "exit 0" }, { - "name": "documented live deployment and verification steps", - "passed": true, - "judgeNotes": "README includes Secret API key creation, matching secret file placement, Compose start/reload, and concrete verification via Prometheus targets." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"project metrics endpoint prometheus\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - }, - { - "url": "https://supabase.com/docs/guides/database/replication/manual-replication-monitoring", - "title": "Manual replication monitoring" - } - ], - "resultChars": 23548 - } - ] + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [], + "loaded": [] }, - "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", - "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/deploy-database-001-prometheus-metrics.json" + "docs": { + "calls": [] + }, + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5-no-skills/build-dataapi-002-restock-alert-report.json" }, { "experiment": "claude-code-sonnet-5-no-skills", @@ -6974,34 +6105,36 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "deploy-functions-001-edge-function-secrets", - "stage": "deploy", + "eval": "build-database-001-migrate-postgres-to-supabase", + "stage": "build", "product": [ - "edge-functions" + "database" ], "topic": [ - "security" + "migrations" ], "suite": "benchmark", "interface": "cli", "passed": true, "checks": [ { - "name": "WEATHER_API_KEY is set as a Function secret on the project", + "name": "all 3 tables exist (teams, members, tasks)", "passed": true }, { - "name": "the weather function is deployed to the project", - "passed": true, - "notes": "status ACTIVE" + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true }, { - "name": "the weather function reads WEATHER_API_KEY from the environment", - "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + "name": "foreign key constraints survived the restore", + "passed": true }, { - "name": "WEATHER_API_KEY value is not committed to the repo", + "name": "tasks_team_status_idx index survived the restore", + "passed": true + }, + { + "name": "sequences synced (next insert won't conflict with existing IDs)", "passed": true } ], @@ -7012,10 +6145,10 @@ "docs": { "calls": [] }, - "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", - "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", + "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", + "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/deploy-functions-001-edge-function-secrets.json" + "sourcePath": "claude-code-sonnet-5-no-skills/build-database-001-migrate-postgres-to-supabase.json" }, { "experiment": "claude-code-sonnet-5-no-skills", @@ -7026,35 +6159,46 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", + "eval": "build-functions-004-service-role-bypass", + "stage": "build", "product": [ - "database", + "edge-functions", "auth", - "storage" + "database" ], "topic": [ - "self-hosting" + "rls", + "security", + "sdk" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", - "passed": true + "name": "rejects missing auth", + "passed": true, + "notes": "status=401" }, { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", - "passed": true + "name": "user A reads own note", + "passed": true, + "notes": "status=200" }, { - "name": "secrets rotated off the shipped defaults", - "passed": true + "name": "reads only with the caller's JWT", + "passed": true, + "notes": "bearer_tokens=2, all_match=true" }, { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", - "passed": true + "name": "user A cannot force-read user B note", + "passed": true, + "notes": "status=200" + }, + { + "name": "user B cannot force-read user A note", + "passed": true, + "notes": "status=200" } ], "skills": { @@ -7064,10 +6208,10 @@ "docs": { "calls": [] }, - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/deploy-self-hosting-001-docker-compose.json" + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5-no-skills/build-functions-004-service-role-bypass.json" }, { "experiment": "claude-code-sonnet-5-no-skills", @@ -7078,49 +6222,67 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", + "eval": "build-functions-005-dual-auth-user-secret", + "stage": "build", "product": [ - "auth" + "edge-functions", + "auth", + "database" ], "topic": [ - "security", - "sdk" + "sdk", + "rls", + "security" ], "suite": "benchmark", - "interface": "mcp", - "passed": false, + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, "checks": [ { - "name": "victim session active before delete-account", - "passed": true + "name": "seed rows present", + "passed": true, + "notes": "found 2/2 seeded rows" }, { - "name": "delete_account flow ran for the victim", - "passed": true + "name": "rejects request with no credentials", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { - "name": "delete-account revokes the user's sessions", + "name": "user with JWT reads only their own rows", "passed": true, - "notes": "sessions left: 0" + "notes": "status 200: {\"data\":[{\"user_id\":\"754f9f21-0dd6-457d-b98e-0b0f2b776e33\",\"metric\":\"steps_a_mssyu18y\",\"value\":111}]}" }, { - "name": "deleted user's refresh token is rejected", - "passed": true + "name": "user cannot read another user's rows by passing user_id", + "passed": true, + "notes": "status 200: {\"data\":[{\"user_id\":\"754f9f21-0dd6-457d-b98e-0b0f2b776e33\",\"metric\":\"steps_a_mssyu18y\",\"value\":111}]}" }, { - "name": "deleted user cannot sign back in", - "passed": false, - "notes": "deleted account can still sign in" + "name": "service key bypasses RLS to read the target user's rows", + "passed": true, + "notes": "status 200: {\"data\":[{\"user_id\":\"7bb39682-d439-4890-8694-0d3e97c5364d\",\"metric\":\"steps_b_mssyu18y\",\"value\":222}]}" }, { - "name": "other users keep their sessions and access", - "passed": true + "name": "non-service key is not granted service access", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { - "name": "diagnosed and explained session revocation", + "name": "rejects an unverified (forged) user token", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "a user token in the apikey slot is not treated as the service key", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "implementation uses @supabase/server", "passed": true, - "judgeNotes": "Meets all rubric requirements: identifies soft-delete/no auth revocation root cause, implements session/refresh-token revocation plus sign-in blocking and RLS enforcement, explains remaining stateless JWT/local-validation window consistently with the RLS fix, and correctly distinguishes publishable vs secret keys and RLS behavior." + "notes": "imports @supabase/server / withSupabase" } ], "skills": { @@ -7128,12 +6290,41 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge functions environment variables SUPABASE_URL SUPABASE_ANON_KEY SUPABASE_SERVICE_ROLE_KEY secret key\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + } + ], + "resultChars": 41771 + } + ] }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "attempts": 2, - "sourcePath": "claude-code-sonnet-5-no-skills/investigate-auth-001-deleted-user-access.json" + "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", + "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/build-functions-005-dual-auth-user-secret.json" }, { "experiment": "claude-code-sonnet-5-no-skills", @@ -7144,13 +6335,14 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "investigate-realtime-001-subscribed-no-events", - "stage": "investigate", + "eval": "build-storage-001-private-bucket-access", + "stage": "build", "product": [ - "realtime", + "storage", "database" ], "topic": [ + "rls", "sdk" ], "suite": "benchmark", @@ -7158,30 +6350,42 @@ "passed": true, "checks": [ { - "name": "orders table added to supabase_realtime publication", + "name": "bucket user-files exists", "passed": true }, { - "name": "courier_locations still in supabase_realtime publication", + "name": "bucket user-files is private", "passed": true }, { - "name": "publication still publishes INSERT events", + "name": "RLS still enabled on storage.objects", "passed": true }, { - "name": "RLS still enabled on orders", + "name": "user A lists only own files", + "passed": true, + "notes": "saw: 01a0005e-1d68-72e9-8597-6c734ea4cf80/receipt-alpha.pdf, 01a0005e-1d68-72e9-8597-6c734ea4cf80/receipt-beta.pdf" + }, + { + "name": "user B cannot read user A files", "passed": true }, { - "name": "staff can still read orders through RLS", - "passed": true, - "notes": "authenticated sees 2 of 2 orders" + "name": "anon reads no files", + "passed": true }, { - "name": "diagnosed missing publication membership", + "name": "user A can upload into own folder", + "passed": true + }, + { + "name": "user B cannot upload into user A folder", + "passed": true + }, + { + "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "The assistant correctly identified that the channel can reach SUBSCRIBED while INSERT events do not fire because public.orders was missing from the supabase_realtime publication. It fixed exactly that with ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, verified courier_locations remained included, and did not disable RLS or weaken policies." + "judgeNotes": "The answer creates a private user-files bucket, keeps RLS enabled, adds authenticated owner-scoped SELECT and INSERT policies using the first path segment as auth.uid(), and provides supabase-js createSignedUrl code with an expiry for temporary sharing. It does not make the bucket public, use public URLs, disable RLS, or use permissive/anon policies." } ], "skills": { @@ -7191,10 +6395,10 @@ "docs": { "calls": [] }, - "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", - "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", + "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", + "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/investigate-realtime-001-subscribed-no-events.json" + "sourcePath": "claude-code-sonnet-5-no-skills/build-storage-001-private-bucket-access.json" }, { "experiment": "claude-code-sonnet-5-no-skills", @@ -7205,32 +6409,33 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "investigate-reliability-003-edge-function-5xx-correlation", - "stage": "investigate", + "eval": "build-tests-001-rls-tenant-isolation", + "stage": "build", "product": [ - "edge-functions" + "database" ], "topic": [ - "observability" + "tests", + "rls" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", "passed": true, "checks": [ { - "name": "identified image-transform and the recurring 503 pattern", + "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "judgeNotes": "The assistant correctly identified `image-transform` as the affected function and described the recurring HTTP 503 pattern across the morning of 2026-04-28, including all 8 failures from 07:00Z through 12:00Z. It also avoided misattributing the main issue to the older `billing-webhook` 503s." + "notes": "1 file(s): supabase/tests/database/01_tenant_isolation.sql" }, { - "name": "attributed recurring 503s to gateway/platform layer, not function code", + "name": "pgTAP isolation tests ran and pass", "passed": true, - "judgeNotes": "Attributes the recurring 503s to gateway/platform/boot-level failure rather than application code, grounded in valid observations: no corresponding execution logs for failed requests, nearby successful invocations, unchanged deployment/version, and distinction from avatar-upload's function-level 500." + "notes": "8 passed, 2 failed" }, { - "name": "recommended a concrete next step", + "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The assistant recommended multiple concrete next steps: checking Edge Function dashboard logs for BOOT_ERROR/WORKER_RESOURCE_LIMIT in a specific time window, reviewing recent deployment img-deploy-42 and considering rollback, investigating resource limits/input size/concurrency, and adding alerting." + "judgeNotes": "Correctly identifies `posts` as the table with broken tenant isolation, explains that authenticated members can read posts from other organizations, and grounds the conclusion in the pgTAP failures. It does not blame `notes` or dismiss the test results." } ], "skills": { @@ -7238,41 +6443,12 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge function 503 error worker boot\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-503-response", - "title": "Edge Function 503 error response" - }, - { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-504-error-response", - "title": "Edge Function 504 error response" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-546-error-response", - "title": "546 - WORKER_RESOURCE_LIMIT Exceeded / WORKER_LIMIT Exceeded" - } - ], - "resultChars": 43344 - } - ] + "calls": [] }, - "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", - "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", + "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", + "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/investigate-reliability-003-edge-function-5xx-correlation.json" + "sourcePath": "claude-code-sonnet-5-no-skills/build-tests-001-rls-tenant-isolation.json" }, { "experiment": "claude-code-sonnet-5-no-skills", @@ -7283,49 +6459,50 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "resolve-dataapi-001-empty-results", - "stage": "resolve", + "eval": "build-vectors-001-rag-with-permissions", + "stage": "build", "product": [ - "data-api", "database", - "auth" + "vectors" ], "topic": [ - "rls", - "sdk" + "sql", + "rls" ], "suite": "benchmark", "interface": "mcp", "passed": true, "checks": [ { - "name": "RLS still enabled on bookmarks", - "passed": true + "name": "document_sections.embedding is vector(384)", + "passed": true, + "notes": "vector(384)" }, { - "name": "user A reads own bookmarks", - "passed": true + "name": "HNSW index on the embedding column", + "passed": true, + "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" }, { - "name": "user B cannot read user A bookmarks", - "passed": true + "name": "index operator class matches the search operator", + "passed": true, + "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" }, { - "name": "anon reads no bookmarks", + "name": "user A search returns only own sections, best match first", "passed": true }, { - "name": "user A can save a new bookmark", + "name": "user B search returns only own sections, best match first", "passed": true }, { - "name": "user B cannot insert a bookmark as user A", + "name": "user A reads only own sections through the API", "passed": true }, { - "name": "diagnosed RLS and added owner-scoped policies", - "passed": true, - "judgeNotes": "Diagnosed RLS deny-all due to no policies, kept RLS enabled, and added authenticated-only SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." + "name": "user A reads only own documents through the API", + "passed": true } ], "skills": { @@ -7335,10 +6512,10 @@ "docs": { "calls": [] }, - "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", - "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", + "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", + "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/resolve-dataapi-001-empty-results.json" + "sourcePath": "claude-code-sonnet-5-no-skills/build-vectors-001-rag-with-permissions.json" }, { "experiment": "claude-code-sonnet-5-no-skills", @@ -7349,42 +6526,30 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "resolve-database-001-migration-history-mismatch", - "stage": "resolve", + "eval": "deploy-database-001-prometheus-metrics", + "stage": "deploy", "product": [ "database" ], "topic": [ - "migrations" + "observability" ], "suite": "benchmark", - "interface": "cli", "passed": true, "checks": [ { - "name": "the avatar_url column is applied on the hosted profiles table", - "passed": true - }, - { - "name": "migration 20240220000000 is recorded in the remote history", - "passed": true - }, - { - "name": "remote migration history matches local migration files", - "passed": true - }, - { - "name": "local migrations are a valid reconciled sequence", + "name": "preserved existing app scrape job", "passed": true }, { - "name": "production profile data is intact (not reset)", - "passed": true + "name": "configured the Supabase Metrics API scrape correctly", + "passed": true, + "judgeNotes": "Meets all requirements: Supabase scrape uses HTTPS, correct metrics path, Basic Auth with password_file, valid project target, app scrape preserved, and docker-compose mounts the secrets directory containing the password file." }, { - "name": "the avatar migration and history reconciliation were done via the Supabase CLI", + "name": "documented live deployment and verification steps", "passed": true, - "judgeNotes": "PASS: avatar_url was applied through `supabase db push` in #14, with output `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration was reconciled by adding the missing local migration file in #12 (`20240115000000_add_profile_bio.sql`), after which `supabase migration list` in #13 showed local and remote histories aligned. psql usage was read-only inspection only; no disallowed workaround seen." + "judgeNotes": "README includes Secret API key creation, matching password_file placement, Compose restart/reload, and concrete verification via Prometheus targets and direct curl." } ], "skills": { @@ -7392,12 +6557,48 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"project metrics endpoint prometheus\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/api/v1-scrape-project-metrics", + "title": "Scrape a project's metrics" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", + "title": "Metrics API" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" + } + ], + "resultChars": 20571 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"create secret api key management api endpoint sb_secret\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [], + "resultChars": 169213 + } + ] }, - "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", - "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", + "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", + "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/resolve-database-001-migration-history-mismatch.json" + "sourcePath": "claude-code-sonnet-5-no-skills/deploy-database-001-prometheus-metrics.json" }, { "experiment": "claude-code-sonnet-5-no-skills", @@ -7408,38 +6609,34 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "resolve-performance-001-slow-query-cpu-spike", - "stage": "resolve", + "eval": "deploy-functions-001-edge-function-secrets", + "stage": "deploy", "product": [ - "database" + "edge-functions" ], "topic": [ - "observability", - "sql" + "security" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", "passed": true, "checks": [ { - "name": "inspected pg_stat_statements for query performance", - "passed": true - }, - { - "name": "ran EXPLAIN on the expensive query", + "name": "WEATHER_API_KEY is set as a Function secret on the project", "passed": true }, { - "name": "created index covering user_id and created_at", - "passed": true + "name": "the weather function is deployed to the project", + "passed": true, + "notes": "status ACTIVE" }, { - "name": "query plan uses an index and avoids sequential scan", + "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." }, { - "name": "inserts still work", + "name": "WEATHER_API_KEY value is not committed to the repo", "passed": true } ], @@ -7450,10 +6647,10 @@ "docs": { "calls": [] }, - "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", - "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", + "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", + "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/resolve-performance-001-slow-query-cpu-spike.json" + "sourcePath": "claude-code-sonnet-5-no-skills/deploy-functions-001-edge-function-secrets.json" }, { "experiment": "claude-code-sonnet-5-no-skills", @@ -7464,54 +6661,34 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "resolve-security-002-rls-cross-tenant-leak", - "stage": "resolve", + "eval": "deploy-self-hosting-001-docker-compose", + "stage": "deploy", "product": [ "database", - "auth" + "auth", + "storage" ], "topic": [ - "rls", - "security" + "self-hosting" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", "passed": true, "checks": [ { - "name": "RLS enabled on notes", - "passed": true - }, - { - "name": "tenant A sees only org A notes", - "passed": true - }, - { - "name": "tenant B cannot read org A notes", - "passed": true - }, - { - "name": "tenant A author can update own note", - "passed": true - }, - { - "name": "tenant B cannot update org A note", - "passed": true - }, - { - "name": "tenant B author can delete own note", + "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", "passed": true }, { - "name": "tenant B cannot delete org A note", + "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", "passed": true }, { - "name": "tenant A can insert note in own org", + "name": "secrets rotated off the shipped defaults", "passed": true }, { - "name": "tenant B cannot insert into org A", + "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", "passed": true } ], @@ -7522,810 +6699,459 @@ "docs": { "calls": [] }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", + "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/resolve-security-002-rls-cross-tenant-leak.json" + "sourcePath": "claude-code-sonnet-5-no-skills/deploy-self-hosting-001-docker-compose.json" }, { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, - "eval": "build-auth-001-email-password-flow", - "stage": "build", + "eval": "investigate-auth-001-deleted-user-access", + "stage": "investigate", "product": [ - "auth", - "database" + "auth" ], "topic": [ - "sdk", - "rls" + "security", + "sdk" ], "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, + "interface": "mcp", + "passed": false, "checks": [ { - "name": "auth module loads and the driver completes", - "passed": true, - "notes": "driver produced a result" - }, - { - "name": "signUp creates the account and returns its user id", - "passed": true, - "notes": "db user 3d73a23b-4cc8-4789-9fd3-9abe325e1baa, signUp returned {\"userId\":\"3d73a23b-4cc8-4789-9fd3-9abe325e1baa\"}" + "name": "victim session active before delete-account", + "passed": true }, { - "name": "signup metadata reaches the profile (display name)", - "passed": true, - "notes": "profiles.display_name = \"Alex Doe\"" + "name": "delete_account flow ran for the victim", + "passed": true }, { - "name": "wrong password is rejected gracefully (no throw, no session)", + "name": "delete-account revokes the user's sessions", "passed": true, - "notes": "{\"error\":\"Invalid login credentials\"}" + "notes": "sessions left: 0" }, { - "name": "signIn with the right password returns the user id", - "passed": true, - "notes": "{\"userId\":\"3d73a23b-4cc8-4789-9fd3-9abe325e1baa\"}" + "name": "deleted user's refresh token is rejected", + "passed": true }, { - "name": "getMyProfile returns the signed-in user's profile", - "passed": true, - "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + "name": "deleted user cannot sign back in", + "passed": false, + "notes": "deleted account can still sign in" }, { - "name": "app code does not use the secret / service-role key", - "passed": true, - "notes": "no secret-key references found" + "name": "other users keep their sessions and access", + "passed": true }, { - "name": "implementation uses @supabase/supabase-js", + "name": "diagnosed and explained session revocation", "passed": true, - "notes": "imports found in: app/src/auth.mjs" + "judgeNotes": "Diagnoses soft-delete-only flow, implements auth/session revocation plus RLS checks to close data access for stale JWTs, explains stateless JWT caveat consistently, and correctly distinguishes publishable vs secret keys including RLS bypass and frontend/server placement." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query {\n searchDocs(query: \"supabase-js signUp signInWithPassword getUser getSession auth admin createUser user_metadata display_name profiles\", limit: 5) {\n nodes {\n title\n href\n content\n ... on ClientLibraryFunctionReference {\n language\n methodName\n }\n }\n totalCount\n }\n}", + "query": "{ searchDocs(query: \"publishable key secret key anon service_role migration RLS\", limit: 5) { nodes { title href content } } }", "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", - "title": "Customizing Emails by Language" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-apple", - "title": "Login with Apple" - } - ], - "resultChars": 79187 + "pages": [], + "resultChars": 149036 }, { "source": "search_docs", - "query": "query {\n searchDocs(query: \"signInWithPassword supabase-js reference signUp options data getUser getSession auth.currentUser\", limit: 10) {\n nodes {\n title\n href\n content\n ... on ClientLibraryFunctionReference {\n language\n methodName\n }\n }\n totalCount\n }\n}", + "query": "{ searchDocs(query: \"publishable and secret API keys\", limit: 2) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/auth-getuser" - }, - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-google", - "title": "Login with Google" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-azure", - "title": "Login with Azure (Microsoft)" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", - "title": "Login with Figma" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" } ], - "resultChars": 193019 + "resultChars": 65638 } ] }, - "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", - "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-auth-001-email-password-flow.json" + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5-no-skills/investigate-auth-001-deleted-user-access.json" }, { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, - "eval": "build-cli-001-bootstrap-app", - "stage": "build", + "eval": "investigate-realtime-001-subscribed-no-events", + "stage": "investigate", "product": [ - "database", - "data-api" + "realtime", + "database" ], "topic": [ - "migrations", - "rls" + "sdk" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "supabase project initialised (supabase/config.toml exists)", + "name": "orders table added to supabase_realtime publication", "passed": true }, { - "name": "todos table is created by a migration file", + "name": "courier_locations still in supabase_realtime publication", "passed": true }, { - "name": "todos table exists with at least 2 seeded rows", - "passed": true, - "notes": "found 2 rows" - }, - { - "name": "row level security is enabled on todos", + "name": "publication still publishes INSERT events", "passed": true }, { - "name": "a SELECT policy targets the authenticated role", + "name": "RLS still enabled on orders", "passed": true }, { - "name": "REST API returns no todos to anonymous requests", + "name": "staff can still read orders through RLS", "passed": true, - "notes": "0 rows" + "notes": "authenticated sees 2 of 2 orders" }, { - "name": "REST API returns the todos to authenticated requests", + "name": "diagnosed missing publication membership", "passed": true, - "notes": "2 rows" + "judgeNotes": "Identifies missing orders table in supabase_realtime publication as root cause, adds public.orders to existing publication, preserves courier_locations/RLS/policies, and does not blame or change client/RLS/networking." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,120p'\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 6745 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"local development migrations RLS select authenticated anon expose table data api\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-flutter", - "title": "Build a User Management App with Flutter" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - } - ], - "resultChars": 152845 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"Securing your API grants authenticated anon RLS select policy\", limit: 3) {\n nodes { title href content }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - } - ], - "resultChars": 63887 - } - ] + "calls": [] }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", + "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", + "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-cli-001-bootstrap-app.json" + "sourcePath": "claude-code-sonnet-5-no-skills/investigate-realtime-001-subscribed-no-events.json" }, { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, - "eval": "build-cli-002-declarative-schema", - "stage": "build", + "eval": "investigate-reliability-003-edge-function-5xx-correlation", + "stage": "investigate", "product": [ - "database" + "edge-functions" ], "topic": [ - "declarative-schema", - "migrations" + "observability" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "supabase db diff used to generate the migration", - "passed": true - }, - { - "name": "schema file updated to include description column", - "passed": true + "name": "identified image-transform and the recurring 503 pattern", + "passed": true, + "judgeNotes": "Identified image-transform as the affected function and described the recurring HTTP 503 pattern across the morning of 2026-04-28, listing the 8 gateway failures from 07:00Z to 12:00Z. Also correctly distinguished unrelated billing-webhook 503s." }, { - "name": "a new migration was generated for the change", - "passed": true + "name": "attributed recurring 503s to gateway/platform layer, not function code", + "passed": true, + "judgeNotes": "Attributes the 503s to gateway/platform before function invocation, supported by absence from edge-function invocation logs, unchanged deployment/version, and distinction from avatar-upload's function-level 500." }, { - "name": "description column exists in the live database", - "passed": true + "name": "recommended a concrete next step", + "passed": true, + "judgeNotes": "The assistant recommended concrete actionable next steps, including checking image-transform capacity/concurrency/CPU limits, adding retries/fallbacks, client-side retry, and decoupling thumbnail generation." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query SearchDocs {\n searchDocs(query: \"declarative database schemas local development migration generate supabase schema_paths\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", - "title": "Declarative database schemas" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/working-with-branches", - "title": "Working with branches" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - } - ], - "resultChars": 65847 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,120p'\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 6745 - } - ] + "calls": [] }, - "prompt": "Add a description text column to the `products` table in my local Supabase stack", - "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-cli-002-declarative-schema.json" + "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", + "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5-no-skills/investigate-reliability-003-edge-function-5xx-correlation.json" }, { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, - "eval": "build-cli-003-pg-cron-queue-workflow", - "stage": "build", + "eval": "resolve-dataapi-001-empty-results", + "stage": "resolve", "product": [ + "data-api", "database", - "edge-functions", - "cron", - "queues" + "auth" ], "topic": [ - "sql", + "rls", "sdk" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", - "passed": true, - "notes": "schedule='* * * * *', active=true" + "name": "RLS still enabled on bookmarks", + "passed": true }, { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 0 -> 1" + "name": "user A reads own bookmarks", + "passed": true }, { - "name": "process-tasks function drains the queue", + "name": "user B cannot read user A bookmarks", + "passed": true + }, + { + "name": "anon reads no bookmarks", + "passed": true + }, + { + "name": "user A can save a new bookmark", + "passed": true + }, + { + "name": "user B cannot insert a bookmark as user A", + "passed": true + }, + { + "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "notes": "function removed the seeded message (id 36) from the queue" + "judgeNotes": "Diagnosed RLS enabled with no policies causing deny-all Data API results, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"pg_cron create cron job every minute run function Supabase local queue pgmq pop delete\", limit: 5) {\n nodes {\n title\n href\n ... on Guide { content }\n ... on Subsection { content }\n ... on CLICommandReference { content }\n ... on ClientLibraryFunctionReference { content }\n ... on TroubleshootingGuide { content }\n ... on ManagementApiReference { content }\n }\n }\n}", - "hasContent": true, - "pages": [] - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"pg_cron create cron job every minute Supabase queue pgmq pop delete\", limit: 10) {\n nodes {\n title\n href\n content\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" - }, - { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", - "title": "pg_cron debugging guide" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_cron", - "title": "pg_cron: Schedule Recurring Jobs with Cron Syntax in Postgres" - }, - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - } - ], - "resultChars": 69751 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"pg_cron unschedule jobname cron.schedule same name idempotent\", limit: 5) {\n nodes { title href content }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/pg_cron-launcher-crashes-with-duplicate-key-value-violates-unique-constraint-cc6472", - "title": "`pg_cron launcher crashes with 'duplicate key value violates unique constraint'`" - }, - { - "url": "https://supabase.com/docs/guides/platform/upgrading", - "title": "Upgrading" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", - "title": "pg_cron debugging guide" - } - ], - "resultChars": 42853 - } - ] + "calls": [] }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", + "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-cli-003-pg-cron-queue-workflow.json" + "sourcePath": "claude-code-sonnet-5-no-skills/resolve-dataapi-001-empty-results.json" }, { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, - "eval": "build-dataapi-001-relational-report", - "stage": "build", + "eval": "resolve-database-001-migration-history-mismatch", + "stage": "resolve", "product": [ - "data-api", "database" ], "topic": [ - "sdk" + "migrations" ], "suite": "benchmark", "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" + "name": "the avatar_url column is applied on the hosted profiles table", + "passed": true }, { - "name": "report numbers match the database (per customer, sorted)", - "passed": true, - "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + "name": "migration 20240220000000 is recorded in the remote history", + "passed": true }, { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table customers" + "name": "remote migration history matches local migration files", + "passed": true }, { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + "name": "local migrations are a valid reconciled sequence", + "passed": true }, { - "name": "report queries via the Data API, not raw SQL", + "name": "production profile data is intact (not reset)", + "passed": true + }, + { + "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "notes": "no psql / raw Postgres driver usage found" + "judgeNotes": "Applied avatar_url via `supabase db push --yes` (#19), which showed `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` Reconciled the orphan bio migration by adding the missing local migration file `supabase/migrations/20240115000000_add_profile_bio.sql` (#17), after which `supabase migration list` showed local and remote matched (#18) and the push succeeded. Only read-only psql inspection was used; no prohibited workaround seen." } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,120p'\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 6745 - } - ] + "calls": [] }, - "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", - "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini/build-dataapi-001-relational-report.json" + "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", + "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/resolve-database-001-migration-history-mismatch.json" }, { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, - "eval": "build-dataapi-002-restock-alert-report", - "stage": "build", + "eval": "resolve-performance-001-slow-query-cpu-spike", + "stage": "resolve", "product": [ - "data-api", "database" ], "topic": [ - "sdk" + "observability", + "sql" ], "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, + "interface": "mcp", + "passed": true, "checks": [ { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" + "name": "inspected pg_stat_statements for query performance", + "passed": true }, { - "name": "alerts match the database (below threshold, sorted)", - "passed": true, - "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + "name": "ran EXPLAIN on the expensive query", + "passed": true }, { - "name": "tables stay locked down (publishable key reads nothing)", - "passed": true, - "notes": "publishable read errored: permission denied for table inventory" + "name": "created index covering user_id and created_at", + "passed": true }, { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + "name": "query plan uses an index and avoids sequential scan", + "passed": true, + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on idx_events_user_id_created_at (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" + "name": "inserts still work", + "passed": true } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase local psql database query cli worker node supabase-js examples\", limit: 5) { edges { node { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on CLICommandReference { title href content } ... on TroubleshootingGuide { title href content } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", - "title": "Testing and linting" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" - }, - { - "url": "https://supabase.com/docs/guides/local-development", - "title": "Local Development & CLI" - }, - { - "url": "https://supabase.com/docs/guides/database/inspect", - "title": "Debugging and monitoring" - } - ], - "resultChars": 36819 - } - ] + "calls": [] }, - "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", - "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini/build-dataapi-002-restock-alert-report.json" + "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", + "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/resolve-performance-001-slow-query-cpu-spike.json" }, { - "experiment": "codex-gpt-5.4-mini", - "experimentSuite": "benchmark", + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" }, - "eval": "build-database-001-migrate-postgres-to-supabase", - "stage": "build", + "eval": "resolve-security-002-rls-cross-tenant-leak", + "stage": "resolve", "product": [ - "database" + "database", + "auth" ], "topic": [ - "migrations" + "rls", + "security" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "all 3 tables exist (teams, members, tasks)", + "name": "RLS enabled on notes", "passed": true }, { - "name": "row counts match (teams=5, members=10, tasks=13)", + "name": "tenant A sees only org A notes", "passed": true }, { - "name": "foreign key constraints survived the restore", + "name": "tenant B cannot read org A notes", "passed": true }, { - "name": "tasks_team_status_idx index survived the restore", + "name": "tenant A author can update own note", "passed": true }, { - "name": "sequences synced (next insert won't conflict with existing IDs)", + "name": "tenant B cannot update org A note", + "passed": true + }, + { + "name": "tenant B author can delete own note", + "passed": true + }, + { + "name": "tenant B cannot delete org A note", + "passed": true + }, + { + "name": "tenant A can insert note in own org", + "passed": true + }, + { + "name": "tenant B cannot insert into org A", "passed": true } ], "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "available": [], + "loaded": [] }, "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,140p'\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 7789 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase db start --from-backup logical backup pg_restore dump\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/download-logical-backups", - "title": "How to download logical backups in Supabase with physical backups enabled?" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", - "title": "Restore a Platform Project to Self-Hosted" - }, - { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/transferring-from-cloud-to-self-host-in-supabase-2oWNvW", - "title": "Transferring from platform to self-hosted Supabase" - }, - { - "url": "https://supabase.com/docs/guides/platform/backups", - "title": "Database Backups" - } - ], - "resultChars": 24731 - } - ] + "calls": [] }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", + "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-database-001-migrate-postgres-to-supabase.json" + "sourcePath": "claude-code-sonnet-5-no-skills/resolve-security-002-rls-cross-tenant-leak.json" }, { "experiment": "codex-gpt-5.4-mini", @@ -8336,46 +7162,60 @@ "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "build-functions-004-service-role-bypass", + "eval": "build-auth-001-email-password-flow", "stage": "build", "product": [ - "edge-functions", "auth", "database" ], "topic": [ - "rls", - "security", - "sdk" + "sdk", + "rls" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", + "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "rejects missing auth", + "name": "auth module loads and the driver completes", "passed": true, - "notes": "status=401" + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user 525000c4-dedc-4742-be9b-8691da52019c, signUp returned {\"userId\":\"525000c4-dedc-4742-be9b-8691da52019c\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" }, { - "name": "user A reads own note", + "name": "signIn with the right password returns the user id", "passed": true, - "notes": "status=200" + "notes": "{\"userId\":\"525000c4-dedc-4742-be9b-8691da52019c\"}" }, { - "name": "reads only with the caller's JWT", + "name": "getMyProfile returns the signed-in user's profile", "passed": true, - "notes": "bearer_tokens=2, all_match=true" + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" }, { - "name": "user A cannot force-read user B note", + "name": "app code does not use the secret / service-role key", "passed": true, - "notes": "status=403" + "notes": "no secret-key references found" }, { - "name": "user B cannot force-read user A note", + "name": "implementation uses @supabase/supabase-js", "passed": true, - "notes": "status=403" + "notes": "imports found in: app/src/auth.mjs" } ], "skills": { @@ -8389,51 +7229,88 @@ }, "docs": { "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,220p'\"", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 12701 + }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions verify_jwt auth.getUser Bearer token\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"auth signUp signIn getUser createClient JavaScript metadata display_name\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" + "url": "https://supabase.com/docs/reference/javascript/auth-signup" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", + "title": "Customizing Emails by Language" }, { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" + "url": "https://supabase.com/docs/reference/dart/auth-signup" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" + "url": "https://supabase.com/docs/reference/swift/auth-signup", + "title": "signUp()" } ], - "resultChars": 32605 + "resultChars": 13813 }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,120p'\"", + "source": "search_docs", + "query": "query { searchDocs(query: \"JavaScript auth signIn password getUser currentSession getSession\", limit: 10) { nodes { __typename ... on ClientLibraryFunctionReference { title href content language methodName } ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/reference/javascript/auth-getsession" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-getuser" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-setsession" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpasskey" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signout" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-refreshsession" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithoauth" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-admin-signout" } ], - "resultChars": 6745 + "resultChars": 17339 } ] }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-functions-004-service-role-bypass.json" + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini/build-auth-001-email-password-flow.json" }, { "experiment": "codex-gpt-5.4-mini", @@ -8444,67 +7321,50 @@ "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "build-functions-005-dual-auth-user-secret", + "eval": "build-cli-001-bootstrap-app", "stage": "build", "product": [ - "edge-functions", - "auth", - "database" + "database", + "data-api" ], "topic": [ - "sdk", - "rls", - "security" + "migrations", + "rls" ], "suite": "benchmark", "interface": "cli", - "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"9c85b93d-750c-4545-ad08-2e86b35ad9a6\",\"metric\":\"steps_a_msj0xq5b\",\"value\":111}]}" + "name": "supabase project initialised (supabase/config.toml exists)", + "passed": true }, { - "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"9c85b93d-750c-4545-ad08-2e86b35ad9a6\",\"metric\":\"steps_a_msj0xq5b\",\"value\":111}]}" + "name": "todos table is created by a migration file", + "passed": true }, { - "name": "service key bypasses RLS to read the target user's rows", + "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"af4aed6d-bffa-426b-98f4-2f78b0fb1835\",\"metric\":\"steps_b_msj0xq5b\",\"value\":222}]}" + "notes": "found 2 rows" }, { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "name": "row level security is enabled on todos", + "passed": true }, { - "name": "rejects an unverified (forged) user token", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "name": "a SELECT policy targets the authenticated role", + "passed": true }, { - "name": "a user token in the apikey slot is not treated as the service key", + "name": "REST API returns no todos to anonymous requests", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "0 rows" }, { - "name": "implementation uses @supabase/server", + "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "imports @supabase/server / withSupabase" + "notes": "2 rows" } ], "skills": { @@ -8519,204 +7379,451 @@ }, "docs": { "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,120p'\"", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 6745 + }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions auth service_role key verify supabase access token request header user_id body\", limit: 5) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", + "query": "query { searchDocs(query: \"local development migrations row level security data api grant authenticated anon select policy\", limit: 5) { nodes { __typename title href content ... on Guide { subsections { nodes { title href content } } } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#before-you-begin", + "title": "Before you begin" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#the-supabase-directory", + "title": "The ./supabase directory" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#move-an-existing-project-to-local-development", + "title": "Move an existing project to local development" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-1-initialize", + "title": "Step 1: Initialize" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-2-authenticate", + "title": "Step 2: Authenticate" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-3-link-to-your-remote-project", + "title": "Step 3: Link to your remote project" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-4-pull-the-remote-schema", + "title": "Step 4: Pull the remote schema" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-5-create-seed-data", + "title": "Step 5: Create seed data" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-6-verify", + "title": "Step 6: Verify" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-7-commit", + "title": "Step 7: Commit" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#start-a-new-project-from-scratch", + "title": "Start a new project from scratch" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-1-initialize-1", + "title": "Step 1: Initialize" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-2-start-the-local-stack", + "title": "Step 2: Start the local stack" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-3-create-your-schema", + "title": "Step 3: Create your schema" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-4-add-seed-data", + "title": "Step 4: Add seed data" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-5-verify", + "title": "Step 5: Verify" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-6-commit", + "title": "Step 6: Commit" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#the-daily-workflow", + "title": "The daily workflow" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#making-schema-changes", + "title": "Making schema changes" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#generating-types", + "title": "Generating types" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#staying-in-sync-with-your-team", + "title": "Staying in sync with your team" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#pushing-to-a-remote-project", + "title": "Pushing to a remote project" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#resetting-a-remote-dev-or-staging-project", + "title": "Resetting a remote dev or staging project" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#key-commands-at-a-glance", + "title": "Key commands at a glance" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#cleaning-up-generated-migrations", + "title": "Cleaning up generated migrations" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#grants", + "title": "Grants" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#revokere-grant-patterns", + "title": "Revoke/re-grant patterns" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#extension-statements", + "title": "Extension statements" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#known-limitations-of-db-diff", + "title": "Known limitations of db diff" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#row-level-security", + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#troubleshooting", + "title": "Troubleshooting" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api#enforce-additional-rules-on-each-request", + "title": "Enforce additional rules on each request" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api#default-privileges-for-new-tables-and-functions", + "title": "Default privileges for new tables and functions" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api#grant-access-explicitly", + "title": "Grant access explicitly" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api#use-a-dedicated-api-schema", + "title": "Use a dedicated API schema" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api#disable-the-data-api", + "title": "Disable the Data API" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api#add-rls-policies", + "title": "Add RLS policies" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api#accessing-request-information", + "title": "Accessing request information" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api#examples", + "title": "Examples" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-flutter", + "title": "Build a User Management App with Flutter" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-flutter#get-api-details", + "title": "Get API details" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-flutter#project-setup", + "title": "Project setup" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-flutter#create-a-project", + "title": "Create a project" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-flutter#set-up-the-database-schema", + "title": "Set up the database schema" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-flutter#building-the-app", + "title": "Building the app" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-flutter#initialize-a-flutter-app", + "title": "Initialize a Flutter app" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-flutter#setup-deep-links", + "title": "Setup deep links" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-flutter#main-function", + "title": "Main function" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-flutter#set-up-a-login-page", + "title": "Set up a login page" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-flutter#set-up-account-page", + "title": "Set up account page" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-flutter#launch", + "title": "Launch!" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-flutter#bonus-profile-photos", + "title": "Bonus: Profile photos" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-flutter#making-sure-we-have-a-public-bucket", + "title": "Making sure we have a public bucket" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-flutter#adding-image-uploading-feature-to-account-page", + "title": "Adding image uploading feature to account page" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-flutter#create-an-upload-widget", + "title": "Create an upload widget" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-flutter#add-the-new-widget", + "title": "Add the new widget" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-flutter#see-also", + "title": "See also" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#example", - "title": "Example" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#creating-policies", + "title": "Creating policies" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#setting-up-auth-context", - "title": "Setting up auth context" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#authenticated-and-unauthenticated-roles", + "title": "Authenticated and unauthenticated roles" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#fetching-the-user", - "title": "Fetching the user" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#auto-enable-rls-for-new-tables", + "title": "Auto-enable RLS for new tables" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#enabling-row-level-security", + "title": "Enabling Row Level Security" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#security-considerations", - "title": "Security considerations" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#policies", + "title": "Policies" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#what-secret-keys-allow-access-to", - "title": "What secret keys allow access to" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#row-level-security-in-supabase", + "title": "Row Level Security in Supabase" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#known-limitations-and-compatibility-differences", - "title": "Known limitations and compatibility differences" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks-2", + "title": "Benchmarks" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#best-practices-for-handling-secret-keys", - "title": "Best practices for handling secret keys" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#add-filters-to-every-query", + "title": "Add filters to every query" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#what-to-do-if-a-secret-key-or-service_role-has-been-leaked-or-compromised", - "title": "What to do if a secret key or service_role has been leaked or compromised?" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks-1", + "title": "Benchmarks" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#overview", - "title": "Overview" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#call-functions-with-select", + "title": "Call functions with select" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#publishable-keys", - "title": "Publishable keys" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks", + "title": "Benchmarks" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys#interaction-with-supabase-auth", - "title": "Interaction with Supabase Auth" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#add-indexes", + "title": "Add indexes" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#rls-performance-recommendations", + "title": "RLS performance recommendations" }, { - "url": "https://supabase.com/docs/guides/functions/auth#public-functions", - "title": "Public functions" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#bypassing-row-level-security", + "title": "Bypassing Row Level Security" }, { - "url": "https://supabase.com/docs/guides/functions/auth#external-webhooks", - "title": "External webhooks" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#mfa", + "title": "MFA" }, { - "url": "https://supabase.com/docs/guides/functions/auth#combining-modes", - "title": "Combining modes" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#authjwt", + "title": "auth.jwt()" }, { - "url": "https://supabase.com/docs/guides/functions/auth#custom-error-responses", - "title": "Custom error responses" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#authuid", + "title": "auth.uid()" }, { - "url": "https://supabase.com/docs/guides/functions/auth#environment-variables", - "title": "Environment variables" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#helper-functions", + "title": "Helper functions" }, { - "url": "https://supabase.com/docs/guides/functions/auth#authenticated-user-calls", - "title": "Authenticated user calls" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#views", + "title": "Views" }, { - "url": "https://supabase.com/docs/guides/functions/auth#service-to-service-calls", - "title": "Service-to-service calls" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#delete-policies", + "title": "DELETE policies" }, { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#update-policies", + "title": "UPDATE policies" }, { - "url": "https://supabase.com/docs/guides/functions/auth-headers#understanding-authorization-headers", - "title": "Understanding authorization headers" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#insert-policies", + "title": "INSERT policies" }, { - "url": "https://supabase.com/docs/guides/functions/auth-headers#the-verify_jwt-platform-check", - "title": "The verify_jwt platform check" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#select-policies", + "title": "SELECT policies" }, { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#more-resources", + "title": "More resources" }, { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#next-steps", - "title": "Next steps" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks-4", + "title": "Benchmarks" }, { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#why-this-pattern-works", - "title": "Why this pattern works" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#specify-roles-in-your-policies", + "title": "Specify roles in your policies" }, { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#browser-client", - "title": "Browser client" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#benchmarks-3", + "title": "Benchmarks" }, { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#edge-function-websocket-proxy", - "title": "Edge Function (WebSocket proxy)" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#minimize-joins", + "title": "Minimize joins" }, { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#database-schema", - "title": "Database schema" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security#use-security-definer-functions", + "title": "Use security definer functions" }, { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets#architecture", - "title": "Architecture" - } - ], - "resultChars": 91747 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase functions serve verify_jwt false config.toml edge function\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" + }, { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs#about-redwoodjs", + "title": "About RedwoodJS" }, { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs#building-the-app", + "title": "Building the app" }, { - "url": "https://supabase.com/docs/guides/functions/deploy", - "title": "Deploy to Production" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs#initialize-a-redwoodjs-app", + "title": "Initialize a RedwoodJS app" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs#app-styling-optional", + "title": "App styling (optional)" }, { - "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", - "title": "Building an MCP Server with mcp-lite" - } - ], - "resultChars": 32331 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions service role key env var SUPABASE_SERVICE_ROLE_KEY secret key Deno.env\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs#start-redwoodjs-and-your-first-page", + "title": "Start RedwoodJS and your first page" + }, { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs#set-up-a-login-component", + "title": "Set up a login component" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs#see-also", + "title": "See also" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs#launch", + "title": "Launch!" }, { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs#create-an-upload-widget", + "title": "Create an upload widget" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", - "title": "Inspecting edge function environment variables" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs#profile-photos", + "title": "Profile photos" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs#update-home-page", + "title": "Update home page" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs#set-up-an-account-component", + "title": "Set up an account component" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs#get-api-details", + "title": "Get API details" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs#set-up-the-database-schema", + "title": "Set up the database schema" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs#create-a-project", + "title": "Create a project" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs#project-setup", + "title": "Project setup" } ], - "resultChars": 44193 + "resultChars": 400793 } ] }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", + "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", + "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-functions-005-dual-auth-user-secret.json" + "sourcePath": "codex-gpt-5.4-mini/build-cli-001-bootstrap-app.json" }, { "experiment": "codex-gpt-5.4-mini", @@ -8727,57 +7834,122 @@ "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "build-storage-001-private-bucket-access", + "eval": "build-cli-002-declarative-schema", "stage": "build", "product": [ - "storage", "database" ], "topic": [ - "rls", - "sdk" + "declarative-schema", + "migrations" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", "passed": true, "checks": [ { - "name": "bucket user-files exists", + "name": "supabase db diff used to generate the migration", "passed": true }, { - "name": "bucket user-files is private", + "name": "schema file updated to include description column", "passed": true }, { - "name": "RLS still enabled on storage.objects", + "name": "a new migration was generated for the change", "passed": true }, { - "name": "user A lists only own files", + "name": "description column exists in the live database", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"declarative database schemas migration generate supabase/schemas\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", + "title": "Declarative database schemas" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, + { + "url": "https://supabase.com/docs/guides/api/rest/generating-types", + "title": "Generating TypeScript Types" + }, + { + "url": "https://supabase.com/docs/guides/api/rest/generating-python-types", + "title": "Generating Python Types" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase", + "title": "Migrating to Supabase" + } + ], + "resultChars": 55971 + } + ] + }, + "prompt": "Add a description text column to the `products` table in my local Supabase stack", + "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini/build-cli-002-declarative-schema.json" + }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-cli-003-pg-cron-queue-workflow", + "stage": "build", + "product": [ + "database", + "edge-functions", + "cron", + "queues" + ], + "topic": [ + "sql", + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", "passed": true, - "notes": "saw: 019fdc8d-2d5c-7238-bdd6-e937eefa55fb/receipt-alpha.pdf, 019fdc8d-2d5c-7238-bdd6-e937eefa55fb/receipt-beta.pdf" - }, - { - "name": "user B cannot read user A files", - "passed": true - }, - { - "name": "anon reads no files", - "passed": true - }, - { - "name": "user A can upload into own folder", - "passed": true + "notes": "schedule='* * * * *', active=true" }, { - "name": "user B cannot upload into user A folder", - "passed": true + "name": "cron command enqueues to the 'tasks' queue", + "passed": true, + "notes": "queue depth 0 -> 1" }, { - "name": "configured private per-user storage access", + "name": "process-tasks function drains the queue", "passed": true, - "judgeNotes": "Creates private user-files bucket, owner-scoped SELECT and INSERT policies for authenticated role using first folder segment = auth.uid(), keeps RLS intact (no disabling), and provides createSignedUrl with a 15-minute expiry. Extra UPDATE/DELETE policies are acceptable." + "notes": "function removed the seeded message (id 5) from the queue" } ], "skills": { @@ -8794,190 +7966,264 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"storage bucket policies objects path owner user id signed URL supabase\", limit: 5) { nodes { __typename title href ... on Guide { content } ... on TroubleshootingGuide { content } ... on ClientLibraryFunctionReference { content language methodName } } } }", + "query": "query { searchDocs(query: \"pg_cron schedule every minute Supabase cron.schedule\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on CLICommandReference { title href content } ... on ManagementApiReference { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" }, { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" }, { - "url": "https://supabase.com/docs/guides/storage", - "title": "Storage" + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", - "title": "Build a User Management App with SvelteKit" + "url": "https://supabase.com/docs/guides/database/extensions/pg_cron", + "title": "pg_cron: Schedule Recurring Jobs with Cron Syntax in Postgres" } ], - "resultChars": 119861 + "resultChars": 50015 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"storage.objects policy owner path prefix foldername user id signed URL createSignedUrl\", limit: 10) { nodes { __typename title href ... on Guide { content } ... on TroubleshootingGuide { content } ... on ClientLibraryFunctionReference { content language methodName } } } }", + "query": "query { searchDocs(query: \"edge functions Deno Supabase deploy local function serve queue postgres\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on CLICommandReference { title href content } ... on ManagementApiReference { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" }, { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", + "title": "Serverless Drivers" }, { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" }, { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" + "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", + "title": "Getting Started with Edge Functions (Dashboard)" }, { - "url": "https://supabase.com/docs/guides/storage/cdn/smart-cdn", - "title": "Smart CDN" - }, + "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", + "title": "Testing and linting" + } + ], + "resultChars": 40187 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"pgmq read pop delete send queue Supabase\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on CLICommandReference { title href content } ... on ManagementApiReference { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/troubleshooting/why-cant-i-uploadlistetc-my-public-bucket-Z6CmGt", - "title": "Why can't I upload/list/etc my public bucket?" + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" }, { - "url": "https://supabase.com/docs/reference/swift/storage-from-createsigneduploadurl", - "title": "from.createSignedUploadURL()" + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" }, { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" }, { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" }, { - "url": "https://supabase.com/docs/reference/swift/storage-from-createsignedurls", - "title": "from.createSignedUrls()" + "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues", + "title": "Expose Queues for local and self-hosted Supabase" } ], - "resultChars": 17609 + "resultChars": 39503 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"storage access control policy examples foldername auth uid owner\", limit: 10) { nodes { __typename title href ... on Guide { content } ... on TroubleshootingGuide { content } } } }", + "query": "query { searchDocs(query: \"cron.unschedule pg_cron schedule name idempotent Supabase\", limit: 5) { nodes { __typename ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/ownership", - "title": "Ownership" + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" }, { - "url": "https://supabase.com/docs/guides/storage/quickstart", - "title": "Storage Quickstart" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" }, { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" }, { - "url": "https://supabase.com/docs/guides/storage/management/copy-move-objects", - "title": "Copy Objects" - }, + "url": "https://supabase.com/docs/guides/cron/install", + "title": "Install" + } + ], + "resultChars": 19123 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"cron.schedule name unschedule pg_cron function signature\", limit: 5) { nodes { __typename ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" }, { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" + "url": "https://supabase.com/docs/guides/cron/quickstart", + "title": "Quickstart" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/guides/cron/install", + "title": "Install" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/supabase-storage-inefficient-folder-operations-and-hierarchical-rls-challenges-b05a4d", - "title": "Supabase Storage: Inefficient folder operations and hierarchical RLS challenges" + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" } ], - "resultChars": 60503 + "resultChars": 25947 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"storage object ownership owner column path prefix foldername access control\", limit: 10) { nodes { __typename title href ... on Guide { content } ... on TroubleshootingGuide { content } } } }", + "query": "query { searchDocs(query: \"verify_jwt edge function config.toml Supabase local\", limit: 5) { nodes { __typename ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/security/ownership", - "title": "Ownership" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/supabase-storage-inefficient-folder-operations-and-hierarchical-rls-challenges-b05a4d", - "title": "Supabase Storage: Inefficient folder operations and hierarchical RLS challenges" - }, - { - "url": "https://supabase.com/docs/guides/storage/management/copy-move-objects", - "title": "Copy Objects" - }, - { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" }, { - "url": "https://supabase.com/docs/guides/storage/schema/design", - "title": "The Storage Schema" + "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", + "title": "Building an MCP Server with mcp-lite" }, { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" + "url": "https://supabase.com/docs/guides/functions/deploy", + "title": "Deploy to Production" }, { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/guides/functions/development-tips", + "title": "Development tips" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/guides/ai-tools/byo-mcp", + "title": "Deploy MCP servers" } ], - "resultChars": 32342 + "resultChars": 31966 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js createSignedUrl storage from bucket expiresIn share file temporary link\", limit: 3) { nodes { __typename title href ... on ClientLibraryFunctionReference { content language methodName } } } }", + "query": "query { searchDocs(query: \"Edge Functions config.toml verify_jwt\", limit: 5) { nodes { __typename ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" }, { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations", - "title": "Manage Storage Image Transformations usage" + "url": "https://supabase.com/docs/guides/functions/deploy", + "title": "Deploy to Production" }, { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" + "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", + "title": "Building an MCP Server with mcp-lite" + }, + { + "url": "https://supabase.com/docs/guides/functions/development-tips", + "title": "Development tips" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" } ], - "resultChars": 1476 + "resultChars": 30528 } ] }, - "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", - "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-storage-001-private-bucket-access.json" + "sourcePath": "codex-gpt-5.4-mini/build-cli-003-pg-cron-queue-workflow.json" + }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-dataapi-001-relational-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini/build-dataapi-001-relational-report.json" }, { "experiment": "codex-gpt-5.4-mini", @@ -8988,33 +8234,44 @@ "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "build-tests-001-rls-tenant-isolation", + "eval": "build-dataapi-002-restock-alert-report", "stage": "build", "product": [ + "data-api", "database" ], "topic": [ - "tests", - "rls" + "sdk" ], "suite": "benchmark", "interface": "cli", - "passed": true, + "cliVersion": "2.109.1", + "passed": false, "checks": [ { - "name": "pgTAP test file(s) written under supabase/tests/", + "name": "report runs and prints JSON", "passed": true, - "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" + "notes": "exit 0" }, { - "name": "pgTAP isolation tests ran and pass", + "name": "alerts match the database (below threshold, sorted)", "passed": true, - "notes": "4 passed, 0 failed" + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" }, { - "name": "agent correctly identifies the posts isolation bug from test results", + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", "passed": true, - "judgeNotes": "The agent correctly identified `posts` as the broken tenant isolation policy, grounded it in the pgTAP result (`cross-org posts` check failed), and did not blame `notes` or dismiss the tests. It also noted `notes` passed and treated the test results as authoritative." + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -9028,80 +8285,12 @@ ] }, "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,160p'\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 1140 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"pgTAP tests RLS Supabase local database tests\", limit: 5) {\n nodes {\n ... on Guide {\n title\n href\n content\n }\n ... on TroubleshootingGuide {\n title\n href\n content\n }\n ... on CLICommandReference {\n title\n href\n content\n }\n }\n totalCount\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/testing/overview", - "title": "Testing Overview" - }, - { - "url": "https://supabase.com/docs/guides/database/testing", - "title": "Testing Your Database" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", - "title": "Testing and linting" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" - } - ], - "resultChars": 68853 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"request.jwt.claims auth.uid set local pgtap Supabase\", limit: 5) {\n nodes {\n ... on Guide {\n title\n href\n content\n }\n }\n totalCount\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/testing/overview", - "title": "Testing Overview" - }, - { - "url": "https://supabase.com/docs/guides/realtime/authorization", - "title": "Realtime Authorization" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/firebase-auth", - "title": "Firebase Auth" - }, - { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" - } - ], - "resultChars": 66662 - } - ] + "calls": [] }, - "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", - "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-tests-001-rls-tenant-isolation.json" + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini/build-dataapi-002-restock-alert-report.json" }, { "experiment": "codex-gpt-5.4-mini", @@ -9112,24 +8301,37 @@ "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "build-vectors-001-rag-with-permissions", + "eval": "build-database-001-migrate-postgres-to-supabase", "stage": "build", "product": [ - "database", - "vectors" + "database" ], "topic": [ - "sql", - "rls" + "migrations" ], "suite": "benchmark", - "interface": "mcp", - "passed": false, + "interface": "cli", + "passed": true, "checks": [ { - "name": "created auth sessions", - "passed": false, - "notes": "Internal server error" + "name": "all 3 tables exist (teams, members, tasks)", + "passed": true + }, + { + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true + }, + { + "name": "foreign key constraints survived the restore", + "passed": true + }, + { + "name": "tasks_team_status_idx index survived the restore", + "passed": true + }, + { + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true } ], "skills": { @@ -9146,66 +8348,66 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"semantic search pgvector row level security documents ownership\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href methodName language } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"local development restore pg_restore dump Supabase\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", + "title": "Migrate from Postgres to Supabase" }, { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" }, { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" }, { - "url": "https://supabase.com/docs/guides/storage/vector/querying-vectors", - "title": "Querying Vectors" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/heroku", + "title": "Migrate from Heroku to Supabase" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pgvector", - "title": "pgvector: Embeddings and vector similarity" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", + "title": "Migrate from Vercel Postgres to Supabase" } ], - "resultChars": 68480 + "resultChars": 42487 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"pgvector extension vector schema extensions match_documents function RLS authenticated ownership\", limit: 10) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href methodName language } } } }", + "query": "query { searchDocs(query: \"supabase start local development database url\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pgvector", - "title": "pgvector: Embeddings and vector similarity" + "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", + "title": "Supabase CLI" }, { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" + "url": "https://supabase.com/docs/guides/deployment/managing-environments", + "title": "Managing Environments" }, { - "url": "https://supabase.com/docs/guides/storage/vector/introduction", - "title": "Vector Buckets" + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" }, { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" + "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", + "title": "Creating a Supabase client for SSR" } ], - "resultChars": 49023 + "resultChars": 113219 } ] }, - "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", - "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini/build-vectors-001-rag-with-permissions.json" + "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", + "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini/build-database-001-migrate-postgres-to-supabase.json" }, { "experiment": "codex-gpt-5.4-mini", @@ -9216,30 +8418,46 @@ "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "deploy-database-001-prometheus-metrics", - "stage": "deploy", + "eval": "build-functions-004-service-role-bypass", + "stage": "build", "product": [ + "edge-functions", + "auth", "database" ], "topic": [ - "observability" + "rls", + "security", + "sdk" ], "suite": "benchmark", - "passed": false, + "interface": "mcp", + "passed": true, "checks": [ { - "name": "preserved existing app scrape job", - "passed": true + "name": "rejects missing auth", + "passed": true, + "notes": "status=401" }, { - "name": "configured the Supabase Metrics API scrape correctly", - "passed": false, - "judgeNotes": "prometheus.yml does not add a deployable Supabase scrape job. It relies on an entrypoint/env injection not shown, uses SUPABASE_SECRET_API_KEY instead of an HTTP Basic Auth password_file, and docker-compose.yml does not mount that password_file via a volume or Compose secret. The existing app job is preserved, but required Supabase scrape wiring is missing." + "name": "user A reads own note", + "passed": true, + "notes": "status=200" }, { - "name": "documented live deployment and verification steps", - "passed": false, - "judgeNotes": "README explains env-based setup and restart, but it does not require placing a matching secret file, and it lacks concrete verification steps via Prometheus targets, PromQL/Grafana, or equivalent." + "name": "reads only with the caller's JWT", + "passed": true, + "notes": "bearer_tokens=2, all_match=true" + }, + { + "name": "user A cannot force-read user B note", + "passed": true, + "notes": "status=200" + }, + { + "name": "user B cannot force-read user A note", + "passed": true, + "notes": "status=200" } ], "skills": { @@ -9256,44 +8474,38 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase metrics Prometheus project metrics\", limit: 5) { nodes { __typename title href content ... on Guide { subsections { nodes { title href content } } } ... on TroubleshootingGuide { } } totalCount } }", - "hasContent": true, - "pages": [] - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"metrics prometheus\", limit: 5) { nodes { title href content } totalCount } }", + "query": "query { searchDocs(query:\"Edge Functions auth getUser verify JWT user data private notes RLS\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href language methodName } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" }, { - "url": "https://supabase.com/docs/guides/database/replication/manual-replication-monitoring", - "title": "Manual replication monitoring" + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" } ], - "resultChars": 23603 + "resultChars": 28361 } ] }, - "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", - "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini/deploy-database-001-prometheus-metrics.json" + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini/build-functions-004-service-role-bypass.json" }, { "experiment": "codex-gpt-5.4-mini", @@ -9304,35 +8516,67 @@ "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "deploy-functions-001-edge-function-secrets", - "stage": "deploy", + "eval": "build-functions-005-dual-auth-user-secret", + "stage": "build", "product": [ - "edge-functions" + "edge-functions", + "auth", + "database" ], "topic": [ + "sdk", + "rls", "security" ], "suite": "benchmark", "interface": "cli", - "passed": true, + "cliVersion": "2.109.1", + "passed": false, "checks": [ { - "name": "WEATHER_API_KEY is set as a Function secret on the project", - "passed": true + "name": "seed rows present", + "passed": true, + "notes": "found 2/2 seeded rows" }, { - "name": "the weather function is deployed to the project", + "name": "rejects request with no credentials", "passed": true, - "notes": "status ACTIVE" + "notes": "status 503: {\"message\":\"name resolution failed\"}" }, { - "name": "the weather function reads WEATHER_API_KEY from the environment", + "name": "user with JWT reads only their own rows", + "passed": false, + "notes": "status 503: {\"message\":\"name resolution failed\"}" + }, + { + "name": "user cannot read another user's rows by passing user_id", + "passed": false, + "notes": "status 503: {\"message\":\"name resolution failed\"}" + }, + { + "name": "service key bypasses RLS to read the target user's rows", + "passed": false, + "notes": "status 503: {\"message\":\"name resolution failed\"}" + }, + { + "name": "non-service key is not granted service access", "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + "notes": "status 503: {\"message\":\"name resolution failed\"}" }, { - "name": "WEATHER_API_KEY value is not committed to the repo", - "passed": true + "name": "rejects an unverified (forged) user token", + "passed": true, + "notes": "status 503: {\"message\":\"name resolution failed\"}" + }, + { + "name": "a user token in the apikey slot is not treated as the service key", + "passed": true, + "notes": "status 503: {\"message\":\"name resolution failed\"}" + }, + { + "name": "implementation uses @supabase/server", + "passed": false, + "notes": "hand-rolled (raw supabase-js or other) — this eval requires @supabase/server" } ], "skills": { @@ -9341,341 +8585,431 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions environment variables secrets deploy supabase secrets set function env var runtime\", limit: 5) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", + "query": "query {\n searchDocs(query: \"Edge Functions auth supabase create function user token apikey service_role verify jwt\", limit: 5) {\n nodes {\n __typename\n ... on Guide { title href content }\n ... on TroubleshootingGuide { title href content }\n ... on CLICommandReference { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on ManagementApiReference { title href content }\n }\n }\n}", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#x-forwarded-headers", - "title": "X-Forwarded headers" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#forwarded-headers-and-cors", - "title": "Forwarded headers and CORS" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#opaque-key-translation", - "title": "Opaque key translation" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#api-key-enforcement-on-protected-routes", - "title": "API key enforcement on protected routes" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#dashboard-basic-auth", - "title": "Dashboard basic auth" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#authentication", - "title": "Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#routes", - "title": "Routes" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#how-the-configuration-is-rendered-at-startup", - "title": "How the configuration is rendered at startup" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#configuration-file-structure", - "title": "Configuration file structure" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#architecture", - "title": "Architecture" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#verify", - "title": "Verify" - }, + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + } + ], + "resultChars": 81572 + }, + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"Edge Functions verify JWT auth getClaims Supabase access token user auth.uid authorization headers\", limit: 10) {\n nodes {\n __typename\n ... on Guide { title href content }\n ... on CLICommandReference { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n }\n }\n}", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#enabling-the-envoy-gateway", - "title": "Enabling the Envoy gateway" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#before-you-begin", - "title": "Before you begin" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#see-also", - "title": "See also" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#common-issues", - "title": "Common issues" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-react", + "title": "Build a User Management App with React" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#logs", - "title": "Logs" + "url": "https://supabase.com/docs/guides/auth/auth-mfa", + "title": "Multi-Factor Authentication" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#troubleshooting", - "title": "Troubleshooting" + "url": "https://supabase.com/docs/guides/auth/jwts", + "title": "JSON Web Token (JWT)" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#admin-interface", - "title": "Admin interface" + "url": "https://supabase.com/docs/guides/auth/jwt-fields", + "title": "JWT Claims Reference" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#customizing-the-configuration", - "title": "Customizing the configuration" + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#security-hardening", - "title": "Security hardening" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#cors", - "title": "CORS" - }, + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + } + ], + "resultChars": 153561 + }, + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"auth getUser token parameter supabase-js getClaims javascript reference\", limit: 10) {\n nodes {\n __typename\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n }\n }\n}", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-react", + "title": "Build a User Management App with React" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#option-2-adopt-the-supabaseserver-sdk", - "title": "Option 2: Adopt the @supabase/server SDK" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-5-verify-nothing-uses-the-legacy-keys", - "title": "Step 5: Verify nothing uses the legacy keys" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", + "title": "Build a User Management App with Angular" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-6-deactivate-the-legacy-keys", - "title": "Step 6: Deactivate the legacy keys" + "url": "https://supabase.com/docs/reference/javascript/auth-getuser" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#known-limitations", - "title": "Known limitations" + "url": "https://supabase.com/docs/reference/javascript/auth-getclaims" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#next-steps", - "title": "Next steps" + "url": "https://supabase.com/docs/guides/auth/jwts", + "title": "JSON Web Token (JWT)" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#option-1-read-the-new-keys-from-the-environment", - "title": "Option 1: Read the new keys from the environment" + "url": "https://supabase.com/docs/reference/swift/auth-getclaims", + "title": "getClaims()" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#before-you-start", - "title": "Before you start" + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", + "title": "OAuth 2.1 Flows" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-1-create-the-new-api-keys", - "title": "Step 1: Create the new API keys" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-2-swap-the-publishable-key-in-client-code", - "title": "Step 2: Swap the publishable key in client code" - }, + "url": "https://supabase.com/docs/guides/auth/jwt-fields", + "title": "JWT Claims Reference" + } + ], + "resultChars": 175488 + }, + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"functions config.toml entrypoint relative path supabase/functions serve entrypoint\", limit: 10) {\n nodes {\n __typename\n ... on Guide { title href content }\n ... on CLICommandReference { title href content }\n }\n }\n}", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-3-swap-the-secret-key-in-backend-code", - "title": "Step 3: Swap the secret key in backend code" + "url": "https://supabase.com/docs/guides/functions/development-tips", + "title": "Development tips" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#database-webhooks-and-pg_net", - "title": "Database Webhooks and pg_net" + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-4-update-edge-functions", - "title": "Step 4: Update Edge Functions" + "url": "https://supabase.com/docs/guides/functions/unit-test", + "title": "Testing your Edge Functions" }, { "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", "title": "Streaming Speech with ElevenLabs" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#run-locally", - "title": "Run locally" + "url": "https://supabase.com/docs/guides/local-development/managing-config", + "title": "Managing config and secrets" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#code-the-supabase-edge-function", - "title": "Code the Supabase Edge Function" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#dependencies", - "title": "Dependencies" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#set-up-the-environment-variables", - "title": "Set up the environment variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#create-a-supabase-edge-function-for-speech-generation", - "title": "Create a Supabase Edge Function for speech generation" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#configure-background-tasks-for-supabase-edge-functions", - "title": "Configure background tasks for Supabase Edge Functions" + "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", + "title": "Building an MCP Server with mcp-lite" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#configure-the-storage-bucket", - "title": "Configure the storage bucket" + "url": "https://supabase.com/docs/guides/functions/wasm", + "title": "Using Wasm modules" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#create-a-supabase-project-locally", - "title": "Create a Supabase project locally" + "url": "https://supabase.com/docs/guides/functions/deploy", + "title": "Deploy to Production" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#setup", - "title": "Setup" + "url": "https://supabase.com/docs/guides/functions/development-environment", + "title": "Development Environment" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#requirements", - "title": "Requirements" - }, + "url": "https://supabase.com/docs/guides/deployment", + "title": "Deployment & Branching" + } + ], + "resultChars": 58445 + }, + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"failed to determine entrypoint edge runtime\" , limit: 5) {\n nodes { __typename ... on TroubleshootingGuide { title href content } ... on Guide { title href content } }\n }\n}", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#introduction", - "title": "Introduction" + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#test-the-function", - "title": "Test the function" + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#set-the-function-secrets", - "title": "Set the function secrets" + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#deploy-to-supabase", - "title": "Deploy to Supabase" + "url": "https://supabase.com/docs/guides/functions/status-codes", + "title": "Status codes" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#try-it-out", - "title": "Try it out" - }, + "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", + "title": "Backup and Restore using the CLI" + } + ], + "resultChars": 87035 + }, + { + "source": "web_search", + "query": "site:github.com/supabase/cli \"failed to determine entrypoint\" \"Edge Functions runtime\"", + "pages": [] + }, + { + "source": "web_search", + "query": "site:supabase.com/docs Edge Functions config.toml function directory entrypoint main.ts user-stats index.ts", + "pages": [] + } + ] + }, + "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", + "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini/build-functions-005-dual-auth-user-secret.json" + }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-storage-001-private-bucket-access", + "stage": "build", + "product": [ + "storage", + "database" + ], + "topic": [ + "rls", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "bucket user-files exists", + "passed": true + }, + { + "name": "bucket user-files is private", + "passed": true + }, + { + "name": "RLS still enabled on storage.objects", + "passed": true + }, + { + "name": "user A lists only own files", + "passed": true, + "notes": "saw: 01a0005e-3d60-72fd-905e-fedbb9c38439/receipt-alpha.pdf, 01a0005e-3d60-72fd-905e-fedbb9c38439/receipt-beta.pdf" + }, + { + "name": "user B cannot read user A files", + "passed": true + }, + { + "name": "anon reads no files", + "passed": true + }, + { + "name": "user A can upload into own folder", + "passed": true + }, + { + "name": "user B cannot upload into user A folder", + "passed": true + }, + { + "name": "configured private per-user storage access", + "passed": true, + "judgeNotes": "Meets rubric: private user-files bucket, RLS enabled, authenticated owner/path-scoped SELECT and INSERT policies with WITH CHECK, and createSignedUrl with expiry for temporary sharing." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"storage policies signed url upload download bucket public private user owner path prefix\", limit: 5) { nodes { __typename title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", - "title": "Transcription Telegram Bot" + "url": "https://supabase.com/docs/guides/troubleshooting/why-cant-i-uploadlistetc-my-public-bucket-Z6CmGt", + "title": "Why can't I upload/list/etc my public bucket?" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#dependencies", - "title": "Dependencies" + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#test-the-bot", - "title": "Test the bot" + "url": "https://supabase.com/docs/guides/storage/serving/downloads", + "title": "Serving assets from Storage" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#set-the-function-secrets", - "title": "Set the function secrets" + "url": "https://supabase.com/docs/guides/storage/cdn/smart-cdn", + "title": "Smart CDN" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#set-up-the-webhook", - "title": "Set up the webhook" - }, + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" + } + ], + "resultChars": 22459 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"storage access control policies storage.objects path like auth.uid example\", limit: 10) { nodes { __typename title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#apply-the-database-migrations", - "title": "Apply the database migrations" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", + "title": "Build a User Management App with Angular" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#deploy-to-supabase", - "title": "Deploy to Supabase" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#code-the-telegram-bot", - "title": "Code the Telegram bot" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-react", + "title": "Build a User Management App with React" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#set-up-the-environment-variables", - "title": "Set up the environment variables" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#create-a-supabase-edge-function-to-handle-telegram-webhook-requests", - "title": "Create a Supabase Edge Function to handle Telegram webhook requests" + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#create-a-database-table-to-log-the-transcription-results", - "title": "Create a database table to log the transcription results" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-expo-react-native", + "title": "Build a User Management App with Expo React Native" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#create-a-supabase-project-locally", - "title": "Create a Supabase project locally" + "url": "https://supabase.com/docs/guides/storage/management/copy-move-objects", + "title": "Copy Objects" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#register-a-telegram-bot", - "title": "Register a Telegram bot" + "url": "https://supabase.com/docs/guides/storage/security/ownership", + "title": "Ownership" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#setup", - "title": "Setup" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-flutter", + "title": "Build a User Management App with Flutter" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#requirements", - "title": "Requirements" - }, + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-refine", + "title": "Build a User Management App with Refine" + } + ], + "resultChars": 222771 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"storage.objects row level security policy owner_id path prefix bucket_id example insert select update\", limit: 10) { nodes { __typename title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#introduction", - "title": "Introduction" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", + "title": "Custom Roles" }, { - "url": "https://supabase.com/docs/guides/functions/secrets#default-secrets", - "title": "Default secrets" + "url": "https://supabase.com/docs/guides/storage/management/copy-move-objects", + "title": "Copy Objects" }, { - "url": "https://supabase.com/docs/guides/functions/secrets#production-secrets", - "title": "Production secrets" + "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", + "title": "Column Level Security" }, { - "url": "https://supabase.com/docs/guides/functions/secrets#local-secrets", - "title": "Local secrets" + "url": "https://supabase.com/docs/guides/storage/schema/design", + "title": "The Storage Schema" }, { - "url": "https://supabase.com/docs/guides/functions/secrets#accessing-environment-variables", - "title": "Accessing environment variables" - } - ], - "resultChars": 149695 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Edge Functions JWT verification no-verify-jwt public browser invoke\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" + "url": "https://supabase.com/docs/guides/troubleshooting/storage-error-403-forbidden-new-row-violates-row-level-security-policy-on-upload-a94384", + "title": "Storage error: 403 Forbidden: 'new row violates row-level security policy' on upload" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/storage/security/ownership", + "title": "Ownership" }, { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" }, { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" } ], - "resultChars": 47787 + "resultChars": 48094 } ] }, - "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", - "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", + "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", + "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/deploy-functions-001-edge-function-secrets.json" + "sourcePath": "codex-gpt-5.4-mini/build-storage-001-private-bucket-access.json" }, { "experiment": "codex-gpt-5.4-mini", @@ -9686,35 +9020,33 @@ "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", + "eval": "build-tests-001-rls-tenant-isolation", + "stage": "build", "product": [ - "database", - "auth", - "storage" + "database" ], "topic": [ - "self-hosting" + "tests", + "rls" ], "suite": "benchmark", "interface": "cli", "passed": true, "checks": [ { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", - "passed": true - }, - { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", - "passed": true + "name": "pgTAP test file(s) written under supabase/tests/", + "passed": true, + "notes": "1 file(s): supabase/tests/tenant-isolation_test.sql" }, { - "name": "secrets rotated off the shipped defaults", - "passed": true + "name": "pgTAP isolation tests ran and pass", + "passed": true, + "notes": "6 passed, 0 failed" }, { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", - "passed": true + "name": "agent correctly identifies the posts isolation bug from test results", + "passed": true, + "judgeNotes": "Identifies `posts` as the tenant isolation flaw: its policy checked membership in any org rather than matching `posts.org_id`. Does not blame `notes` or dismiss tests; reports pgTAP verification after fixing." } ], "skills": { @@ -9730,25 +9062,87 @@ "docs": { "calls": [ { - "source": "web_search", - "query": "site:supabase.com/changelog.md Supabase breaking-change self-hosting docker compose", - "pages": [] + "source": "search_docs", + "query": "query {\n searchDocs(query: \"Supabase testing RLS policies pgTAP local database tests auth.uid set_config request.jwt.claim.sub\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/overview", + "title": "Testing Overview" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" + } + ], + "resultChars": 89166 }, { - "source": "web_search", - "query": "https://supabase.com/docs/guides/self-hosting/docker", + "source": "search_docs", + "query": "query {\n searchDocs(query: \"request.jwt.claims auth.uid current_setting Supabase RLS tests set local role authenticated\", limit: 10) {\n nodes {\n title\n href\n content\n }\n }\n}", + "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/docker" + "url": "https://supabase.com/docs/guides/realtime/authorization", + "title": "Realtime Authorization" + }, + { + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/overview", + "title": "Testing Overview" + }, + { + "url": "https://supabase.com/docs/guides/auth/third-party/firebase-auth", + "title": "Firebase Auth" + }, + { + "url": "https://supabase.com/docs/guides/realtime/postgres-changes", + "title": "Postgres Changes" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/auth/third-party/auth0", + "title": "Auth0" + }, + { + "url": "https://supabase.com/docs/guides/auth/third-party/workos", + "title": "WorkOS" + }, + { + "url": "https://supabase.com/docs/guides/auth/users", + "title": "Users" } - ] + ], + "resultChars": 143969 } ] }, - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", + "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", + "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/deploy-self-hosting-001-docker-compose.json" + "sourcePath": "codex-gpt-5.4-mini/build-tests-001-rls-tenant-isolation.json" }, { "experiment": "codex-gpt-5.4-mini", @@ -9759,48 +9153,50 @@ "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", + "eval": "build-vectors-001-rag-with-permissions", + "stage": "build", "product": [ - "auth" + "database", + "vectors" ], "topic": [ - "security", - "sdk" + "sql", + "rls" ], "suite": "benchmark", "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { - "name": "victim session active before delete-account", - "passed": true + "name": "document_sections.embedding is vector(384)", + "passed": true, + "notes": "vector(384)" }, { - "name": "delete_account flow ran for the victim", - "passed": true + "name": "HNSW index on the embedding column", + "passed": true, + "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops) WHERE (embedding IS NOT NULL)" }, { - "name": "delete-account revokes the user's sessions", + "name": "index operator class matches the search operator", "passed": true, - "notes": "sessions left: 0" + "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops) WHERE (embedding IS NOT NULL)" }, { - "name": "deleted user's refresh token is rejected", - "passed": true + "name": "user A search returns only own sections, best match first", + "passed": false }, { - "name": "deleted user cannot sign back in", - "passed": true + "name": "user B search returns only own sections, best match first", + "passed": false }, { - "name": "other users keep their sessions and access", + "name": "user A reads only own sections through the API", "passed": true }, { - "name": "diagnosed and explained session revocation", - "passed": true, - "judgeNotes": "Meets the rubric: identifies soft-delete-only bug, updates flow to delete auth user and lock down RLS, explains stale JWT caveat consistently with the RLS mitigation, and correctly distinguishes publishable/anon vs secret/service_role keys." + "name": "user A reads only own documents through the API", + "passed": true } ], "skills": { @@ -9817,92 +9213,80 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"delete user revoke sessions auth admin sign out user delete account\", limit: 5) { nodes { title href ... on Guide { content } ... on TroubleshootingGuide { content } ... on ClientLibraryFunctionReference { methodName language content href } } } }", + "query": "query { searchDocs(query: \"pgvector semantic search Supabase RPC match_documents row level security\", limit: 5) { nodes { __typename title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" + "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", + "title": "Semantic Search" }, { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-signout" + "url": "https://supabase.com/docs/guides/ai/hybrid-search", + "title": "Hybrid search" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-signout" + "url": "https://supabase.com/docs/guides/database/full-text-search", + "title": "Full Text Search" } ], - "resultChars": 20977 + "resultChars": 88003 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"publishable key secret key anon key service role frontend rls\", limit: 5) { nodes { title href ... on Guide { content } ... on ClientLibraryFunctionReference { methodName language content href } } } }", + "query": "query { searchDocs(query: \"Supabase row level security documents owner_id policy authenticated select insert update\", limit: 5) { nodes { __typename title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/auth/users", + "title": "Users" } ], - "resultChars": 68789 - }, - { - "source": "web_search", - "query": "site:supabase.com/changelog.md Supabase changelog markdown", - "pages": [] + "resultChars": 46162 }, { "source": "web_search", "query": "https://supabase.com/changelog.md", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs/guides/getting-started/api-keys Supabase publishable secret key anon service_role", - "pages": [] - }, - { - "source": "web_search", - "query": "https://supabase.com/docs/guides/getting-started/api-keys", "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/api-keys" + "url": "https://supabase.com/changelog.md" } ] }, { "source": "web_search", - "query": "'Deleting users' in https://supabase.com/docs/guides/auth/managing-user-data", + "query": "site:supabase.com/changelog.md Supabase changelog markdown", "pages": [] } ] }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", + "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini/investigate-auth-001-deleted-user-access.json" + "sourcePath": "codex-gpt-5.4-mini/build-vectors-001-rag-with-permissions.json" }, { "experiment": "codex-gpt-5.4-mini", @@ -9913,44 +9297,30 @@ "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "investigate-realtime-001-subscribed-no-events", - "stage": "investigate", + "eval": "deploy-database-001-prometheus-metrics", + "stage": "deploy", "product": [ - "realtime", "database" ], "topic": [ - "sdk" + "observability" ], "suite": "benchmark", - "interface": "mcp", - "passed": true, + "passed": false, "checks": [ { - "name": "orders table added to supabase_realtime publication", - "passed": true - }, - { - "name": "courier_locations still in supabase_realtime publication", - "passed": true - }, - { - "name": "publication still publishes INSERT events", - "passed": true - }, - { - "name": "RLS still enabled on orders", + "name": "preserved existing app scrape job", "passed": true }, { - "name": "staff can still read orders through RLS", - "passed": true, - "notes": "authenticated sees 2 of 2 orders" + "name": "configured the Supabase Metrics API scrape correctly", + "passed": false, + "judgeNotes": "Fails: Supabase scrape is generated in docker-compose with inline basic_auth password from SUPABASE_METRICS_API_KEY, not password_file; docker-compose does not mount a password_file via volume or Compose secret; README instructs using a Secret API key. App scrape is preserved and endpoint/HTTPS/project target are otherwise present." }, { - "name": "diagnosed missing publication membership", - "passed": true, - "judgeNotes": "Identified the root cause as orders missing from supabase_realtime despite successful subscription, fixed by ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, and preserved courier_locations plus RLS/policies." + "name": "documented live deployment and verification steps", + "passed": false, + "judgeNotes": "README explains creating a Supabase Secret API key and restarting the Compose stack, but it does not require placing a matching secret file, and it lacks concrete verification steps via Prometheus targets, PromQL/Grafana, or equivalent." } ], "skills": { @@ -9966,452 +9336,697 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Realtime postgres_changes publication supabase_realtime table not receiving events\", limit: 5) { nodes { __typename title href content ... on Guide { subsections { nodes { title href content } } } } } }", + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | sed -n '1,220p'\"", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", - "title": "Subscribing to Database Changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#broadcast-authorization", - "title": "Broadcast authorization" - }, - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#streaming-updates", - "title": "Streaming updates" - }, - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#streaming-inserts", - "title": "Streaming inserts" - }, - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#enable-postgres-changes", - "title": "Enable Postgres Changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#using-postgres-changes", - "title": "Using Postgres Changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#listening-on-client-side", - "title": "Listening on client side" - }, - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#create-a-trigger", - "title": "Create a trigger" - }, - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#create-a-trigger-function", - "title": "Create a trigger function" - }, - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes#using-broadcast", - "title": "Using Broadcast" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks", - "title": "Benchmarks" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#methodology", - "title": "Methodology" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#workloads", - "title": "Workloads" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#results", - "title": "Results" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#broadcast-using-websockets", - "title": "Broadcast: Using WebSockets" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#broadcast-using-the-database", - "title": "Broadcast: Using the database" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#broadcast-impact-of-payload-size", - "title": "Broadcast: Impact of payload size" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#1kb-payload", - "title": "1KB payload" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#10kb-payload", - "title": "10KB payload" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#50kb-payload", - "title": "50KB payload" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#broadcast-scalability-scenarios", - "title": "Broadcast: Scalability scenarios" - }, + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 12701 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"metrics prometheus observability\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on CLICommandReference { title href content } ... on ManagementApiReference { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } totalCount } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#realtime-auth", - "title": "Realtime Auth" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", + "title": "Metrics API" }, { - "url": "https://supabase.com/docs/guides/realtime/benchmarks#postgres-changes", - "title": "Postgres Changes" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes", - "title": "Postgres Changes" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#refreshed-tokens", - "title": "Refreshed tokens" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#custom-tokens", - "title": "Custom tokens" - }, + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" + } + ], + "resultChars": 27599 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"postgres metrics endpoint prometheus\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on CLICommandReference { title href content } ... on ManagementApiReference { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } totalCount } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#private-schemas", - "title": "Private schemas" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", + "title": "Metrics API" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#receiving-old-records", - "title": "Receiving old records" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#quick-start", - "title": "Quick start" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#usage", - "title": "Usage" - }, + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" + } + ], + "resultChars": 20098 + } + ] + }, + "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", + "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini/deploy-database-001-prometheus-metrics.json" + }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "deploy-functions-001-edge-function-secrets", + "stage": "deploy", + "product": [ + "edge-functions" + ], + "topic": [ + "security" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "WEATHER_API_KEY is set as a Function secret on the project", + "passed": true + }, + { + "name": "the weather function is deployed to the project", + "passed": true, + "notes": "status ACTIVE" + }, + { + "name": "the weather function reads WEATHER_API_KEY from the environment", + "passed": true, + "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + }, + { + "name": "WEATHER_API_KEY value is not committed to the repo", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Functions secrets environment variables deploy function get secret from env Deno.serve supabase secrets set function invoke\", limit: 5) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } ... on CLICommandReference { title href content } ... on ManagementApiReference { title href content } } totalCount } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#listening-to-specific-schemas", - "title": "Listening to specific schemas" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#listening-to-insert-events", - "title": "Listening to INSERT events" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#custom-env-vars-not-available-in-functions", + "title": "Custom env vars not available in functions" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#listening-to-update-events", - "title": "Listening to UPDATE events" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#changes-to-function-code-not-reflected-after-editing", + "title": "Changes to function code not reflected after editing" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#listening-to-delete-events", - "title": "Listening to DELETE events" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#500-error-on-invocation", + "title": "500 error on invocation" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#listening-to-specific-tables", - "title": "Listening to specific tables" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#troubleshooting", + "title": "Troubleshooting" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#listening-to-multiple-changes", - "title": "Listening to multiple changes" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#copying-functions-from-supabase-platform", + "title": "Copying functions from Supabase platform" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#filtering-for-specific-changes", - "title": "Filtering for specific changes" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#deploying-functions-to-a-remote-server", + "title": "Deploying functions to a remote server" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#available-filters", - "title": "Available filters" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#managing-functions-via-dashboard", + "title": "Managing functions via dashboard" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#equal-to-eq", - "title": "Equal to (eq)" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#internal-vs-external-urls", + "title": "Internal vs external URLs" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#not-equal-to-neq", - "title": "Not equal to (neq)" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#calling-supabase-services-from-functions", + "title": "Calling Supabase services from functions" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#less-than-lt", - "title": "Less than (lt)" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#accessing-variables-in-functions", + "title": "Accessing variables in functions" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#less-than-or-equal-to-lte", - "title": "Less than or equal to (lte)" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#using-inline-environment-variables", + "title": "Using inline environment variables" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#greater-than-gt", - "title": "Greater than (gt)" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#using-an-env-file-recommended", + "title": "Using an env file (recommended)" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#greater-than-or-equal-to-gte", - "title": "Greater than or equal to (gte)" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#custom-environment-variables", + "title": "Custom environment variables" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#contained-in-list-in", - "title": "Contained in list (in)" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#step-3-invoke-your-function", + "title": "Step 3: Invoke your function" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#database-instance-and-realtime-performance", - "title": "Database instance and realtime performance" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#step-2-restart-the-functions-service-to-pick-up-the-new-function", + "title": "Step 2: Restart the functions service to pick up the new function" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#spaces-in-table-names", - "title": "Spaces in table names" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#step-1-add-a-new-function-directory-and-the-function-code", + "title": "Step 1: Add a new function directory and the function code" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#delete-events-are-not-filterable", - "title": "Delete events are not filterable" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#create-a-new-function", + "title": "Create a new function" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes#limitations", - "title": "Limitations" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#invoke-the-default-function", + "title": "Invoke the default function" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol", - "title": "Realtime Protocol" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#memory-or-timeout-errors", + "title": "Memory or timeout errors" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#join-errors", - "title": "Join errors" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#websocket-connection-setup", - "title": "WebSocket connection setup" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#how-do-we-handle-retries", + "title": "How do we handle retries?" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#protocol-messages", - "title": "Protocol messages" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#how-do-visibility-timeouts-work", + "title": "How do visibility timeouts work?" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#100", - "title": "1.0.0" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#why-queue-requests-instead-of-processing-them-immediately", + "title": "Why queue requests instead of processing them immediately?" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#200", - "title": "2.0.0" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#why-not-one-request-per-row", + "title": "Why not one request per row?" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#text-frames", - "title": "Text frames" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#why-not-generate-all-embeddings-in-a-single-edge-function-request", + "title": "Why not generate all embeddings in a single Edge Function request?" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#binary-frames", - "title": "Binary frames" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#step-2-create-utility-functions", + "title": "Step 2: Create utility functions" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#user-broadcast-push", - "title": "User Broadcast Push" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#step-1-enable-extensions", + "title": "Step 1: Enable extensions" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#user-broadcast", - "title": "User Broadcast" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#implementation", + "title": "Implementation" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#event-types", - "title": "Event types" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#understanding-the-architecture", + "title": "Understanding the architecture" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#client-sent-events", - "title": "Client sent events" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#understanding-the-challenge", + "title": "Understanding the challenge" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#phx_join", - "title": "phx_join" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#step-3-create-queue-and-triggers", + "title": "Step 3: Create queue and triggers" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#phx_leave", - "title": "phx_leave" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#see-also", + "title": "See also" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#heartbeat", - "title": "heartbeat" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#conclusion", + "title": "Conclusion" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#access_token", - "title": "access_token" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#troubleshooting", + "title": "Troubleshooting" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#broadcast-text-frame", - "title": "broadcast (text frame)" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#3-insert-and-update-documents", + "title": "3. Insert and update documents" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#broadcast-binary-frame", - "title": "broadcast (binary frame)" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#optional-clearing-embeddings-on-update", + "title": "(Optional) Clearing embeddings on update" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#presence", - "title": "presence" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#2-create-triggers-to-enqueue-embedding-jobs", + "title": "2. Create triggers to enqueue embedding jobs" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#server-sent-events", - "title": "Server sent events" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#1-create-table-to-store-documents-with-embeddings", + "title": "1. Create table to store documents with embeddings" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#phx_close", - "title": "phx_close" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#usage", + "title": "Usage" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#phx_error", - "title": "phx_error" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#step-4-create-the-edge-function", + "title": "Step 4: Create the Edge Function" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#phx_reply", - "title": "phx_reply" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings#is-10-seconds-a-good-interval-for-processing", + "title": "Is 10 seconds a good interval for processing?" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#system", - "title": "system" + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#broadcast-text-frame-1", - "title": "broadcast (text frame)" + "url": "https://supabase.com/docs/guides/functions/secrets#accessing-environment-variables", + "title": "Accessing environment variables" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#broadcast-binary-frame-1", - "title": "broadcast (binary frame)" + "url": "https://supabase.com/docs/guides/functions/secrets#default-secrets", + "title": "Default secrets" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#postgres_changes", - "title": "postgres_changes" + "url": "https://supabase.com/docs/guides/functions/secrets#production-secrets", + "title": "Production secrets" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#presence_state", - "title": "presence_state" + "url": "https://supabase.com/docs/guides/functions/secrets#local-secrets", + "title": "Local secrets" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#presence_diff", - "title": "presence_diff" + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#error-handling", - "title": "Error handling" + "url": "https://supabase.com/docs/guides/functions#quick-technical-notes", + "title": "Quick technical notes" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#channel-level-system-errors", - "title": "Channel-level system errors" + "url": "https://supabase.com/docs/guides/functions#when-to-use-edge-functions", + "title": "When to use Edge Functions" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#postgres-changes-subscription-errors", - "title": "Postgres Changes subscription errors" + "url": "https://supabase.com/docs/guides/functions#examples", + "title": "Examples" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#broadcast-errors", - "title": "Broadcast errors" + "url": "https://supabase.com/docs/guides/functions#how-it-works", + "title": "How it works" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#presence-errors", - "title": "Presence errors" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#401-invalid-jwt" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol#access-token-refresh", - "title": "Access token refresh" - }, + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#400-missing-function-name-in-request" + } + ], + "resultChars": 134993 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Functions invoke URL Authorization Bearer anon key functions/v1 project route\", limit: 10) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } } totalCount } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/realtime/protocol#reconnection", - "title": "Reconnection" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started", - "title": "Getting Started with Realtime" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#when-to-use-presence", - "title": "When to use Presence" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#quick-start", - "title": "Quick start" + "url": "https://supabase.com/docs/guides/functions/recursive-functions", + "title": "Recursive / Nested Function Calls" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#framework-examples", - "title": "Framework examples" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#integration-guides", - "title": "Integration guides" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#advanced-topics", - "title": "Advanced topics" + "url": "https://supabase.com/docs/guides/functions/status-codes", + "title": "Status codes" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#security--configuration", - "title": "Security & configuration" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#core-features", - "title": "Core features" + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#next-steps", - "title": "Next steps" - }, + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + } + ], + "resultChars": 111846 + } + ] + }, + "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", + "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini/deploy-functions-001-edge-function-secrets.json" + }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "deploy-self-hosting-001-docker-compose", + "stage": "deploy", + "product": [ + "database", + "auth", + "storage" + ], + "topic": [ + "self-hosting" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", + "passed": true + }, + { + "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", + "passed": true + }, + { + "name": "secrets rotated off the shipped defaults", + "passed": true + }, + { + "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"self-hosting docker compose env config supabase docker compose\", limit: 5) { edges { node { ... on Guide { title href content } ... on CLICommandReference { title href content } ... on TroubleshootingGuide { title href content } } } totalCount } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/realtime/getting_started#when-to-use-postgres-changes", - "title": "When to use Postgres Changes" + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#when-to-use-broadcast", - "title": "When to use Broadcast" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#choose-the-right-feature", - "title": "Choose the right feature" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#clean-up-subscriptions", - "title": "Clean up subscriptions" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#follow-naming-conventions", - "title": "Follow naming conventions" - }, + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https", + "title": "Configure Reverse Proxy and HTTPS" + } + ], + "resultChars": 94582 + } + ] + }, + "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", + "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini/deploy-self-hosting-001-docker-compose.json" + }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "investigate-auth-001-deleted-user-access", + "stage": "investigate", + "product": [ + "auth" + ], + "topic": [ + "security", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": false, + "checks": [ + { + "name": "victim session active before delete-account", + "passed": true + }, + { + "name": "delete_account flow ran for the victim", + "passed": true + }, + { + "name": "delete-account revokes the user's sessions", + "passed": false, + "notes": "sessions left: 1" + }, + { + "name": "deleted user's refresh token is rejected", + "passed": false, + "notes": "refresh token still produces a session" + }, + { + "name": "deleted user cannot sign back in", + "passed": false, + "notes": "deleted account can still sign in" + }, + { + "name": "other users keep their sessions and access", + "passed": true + }, + { + "name": "diagnosed and explained session revocation", + "passed": false, + "judgeNotes": "Failed: the fix leaves delete_account as a profile soft-delete and only tightens RLS. It does not delete the auth user or remove/revoke identities, sessions, or refresh tokens, so it does not implement the required real auth revocation." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"delete user revoke sessions auth.admin.deleteUser auth.sessions\", limit: 5) { nodes { __typename title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/realtime/getting_started#use-private-channels", - "title": "Use private channels" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#essential-best-practices", - "title": "Essential best practices" + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#53-using-database-triggers", - "title": "5.3 using database triggers" + "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#52-using-httprest-api", - "title": "5.2 using HTTP/REST API" + "url": "https://supabase.com/docs/reference/swift/auth-admin-deleteuser", + "title": "deleteUser()" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#51-using-client-libraries", - "title": "5.1 using client libraries" - }, + "url": "https://supabase.com/docs/reference/swift/auth-admin-signout", + "title": "signOut()" + } + ], + "resultChars": 8850 + }, + { + "source": "web_search", + "query": "site:supabase.com/docs publishable secret keys frontend anon service_role Supabase", + "pages": [] + }, + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ { - "url": "https://supabase.com/docs/guides/realtime/getting_started#5-send-and-receive-messages", - "title": "5. Send and receive messages" - }, + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "web_search", + "query": "site:supabase.com/changelog.md supabase changelog md", + "pages": [] + } + ] + }, + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini/investigate-auth-001-deleted-user-access.json" + }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "investigate-realtime-001-subscribed-no-events", + "stage": "investigate", + "product": [ + "realtime", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "orders table added to supabase_realtime publication", + "passed": true + }, + { + "name": "courier_locations still in supabase_realtime publication", + "passed": true + }, + { + "name": "publication still publishes INSERT events", + "passed": true + }, + { + "name": "RLS still enabled on orders", + "passed": true + }, + { + "name": "staff can still read orders through RLS", + "passed": true, + "notes": "authenticated sees 2 of 2 orders" + }, + { + "name": "diagnosed missing publication membership", + "passed": true, + "judgeNotes": "The assistant correctly identified that the subscription can reach SUBSCRIBED while orders INSERT events are not delivered because public.orders was missing from the supabase_realtime publication. It fixed exactly that with ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, preserved courier_locations in the publication, and did not disable RLS or alter policies." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"postgres_changes Realtime publication RLS insert events table must be in supabase_realtime publication\", limit: 5) { nodes { __typename title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/realtime/getting_started#4-set-up-authorization", - "title": "4. Set up authorization" + "url": "https://supabase.com/docs/guides/realtime/benchmarks", + "title": "Benchmarks" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#3-create-your-first-channel", - "title": "3. Create your first Channel" + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", + "title": "Subscribing to Database Changes" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#get-api-details", - "title": "Get API details" + "url": "https://supabase.com/docs/guides/realtime/postgres-changes", + "title": "Postgres Changes" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#2-initialize-the-client", - "title": "2. Initialize the client" + "url": "https://supabase.com/docs/guides/realtime/authorization", + "title": "Realtime Authorization" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started#1-install-the-client-library", - "title": "1. Install the client library" + "url": "https://supabase.com/docs/guides/realtime/architecture", + "title": "Realtime Architecture" } ], - "resultChars": 389763 + "resultChars": 84080 } ] }, @@ -10444,17 +10059,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform as the affected function and described repeated HTTP 503 gateway responses throughout the morning of 2026-04-28 from 07:00Z to 12:00Z, while distinguishing unrelated issues." + "judgeNotes": "Identified image-transform as the affected function and described the recurring intermittent 503 gateway-layer failures throughout the morning of 2026-04-28, covering most of the expected failures across the 07:00Z-12:00Z window." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "Attributes recurring image-transform 503s to the edge/API gateway/platform layer rather than function code, and grounds this in API/gateway 503s alongside successful edge-function executions, plus distinguishes the avatar-upload 500 as a separate function-level error." + "judgeNotes": "Attributes the recurring image-transform 503s to the gateway/Edge Function availability/routing layer, grounded in the observation that 503s lack matching edge-function execution logs while nearby 200s succeed. It also distinguishes the avatar-upload 500 as a separate function-level error." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended concrete next steps, including pulling fuller traces for failing request windows, checking Supabase edge/runtime incidents or rollouts, adding retry/backoff, and investigating a specific isolated 500 code path." + "judgeNotes": "The assistant provided specific actionable next steps, including inspecting function/package code, correlating failures with cold starts/deploys/instance recycling, adding retry backoff, capturing the stack trace for a specific 500, and further investigating deployment history." } ], "skills": { @@ -10525,7 +10140,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies as deny-all for Data API, kept RLS enabled, created authenticated SELECT policy scoped to user_id = auth.uid(), and created authenticated INSERT policy with WITH CHECK enforcing user_id = auth.uid()." + "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies as deny-all for the Data API, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid(), with INSERT enforced via WITH CHECK." } ], "skills": { @@ -10542,7 +10157,7 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Exposing a Table to the Data API RLS bookmarks anon authenticated GRANT\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on CLICommandReference { title href content } } } }", + "query": "query { searchDocs(query: \"Exposing a Table to the Data API grants RLS bookmarks authenticated anon\", limit: 5) { nodes { title href ... on Guide { content } } } }", "hasContent": true, "pages": [ { @@ -10553,10 +10168,6 @@ "url": "https://supabase.com/docs/guides/database/secure-data", "title": "Securing your data" }, - { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0017_foreign_table_in_api", - "title": "Database Advisor: Lint 0017_foreign_table_in_api" - }, { "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", "title": "Row Level Security" @@ -10564,9 +10175,13 @@ { "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0013_rls_disabled_in_public", "title": "Database Advisor: Lint 0013_rls_disabled_in_public" + }, + { + "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0017_foreign_table_in_api", + "title": "Database Advisor: Lint 0017_foreign_table_in_api" } ], - "resultChars": 58013 + "resultChars": 57888 } ] }, @@ -10619,7 +10234,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "Applied avatar_url via `supabase db push --db-url \"$DB_URL\"` (#42), which showed `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` Reconciled orphan bio history by adding local migration `supabase/migrations/20240115000000_add_profile_bio.sql` (#36-37), after which `supabase migration list --db-url` showed local/remote aligned (#39, #45). Only read-only psql inspection observed; no prohibited workaround." + "judgeNotes": "Avatar migration was applied through `supabase db push --db-url \"$(cat supabase/.temp/pooler-url)\" --yes`, which output `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration was reconciled by adding local file `supabase/migrations/20240115000000_add_profile_bio.sql`; `supabase migration list --db-url ...` then showed it matched remote. No disallowed workaround was used; only read-only psql inspection occurred." } ], "skills": { @@ -10677,7 +10292,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", @@ -10698,23 +10313,31 @@ "calls": [ { "source": "search_docs", - "query": "query {\n searchDocs(query: \"composite index query optimization order by desc limit user_id created_at\", limit: 5) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n ... on ManagementApiReference { title href content }\n }\n }\n}", + "query": "query { searchDocs(query: \"query optimization index recent events user_id created_at\", limit: 5) { nodes { __typename title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" + "url": "https://supabase.com/docs/guides/storage/analytics/query-with-postgres", + "title": "Query with Postgres" + }, + { + "url": "https://supabase.com/docs/guides/storage/debugging/logs", + "title": "Logs" }, { "url": "https://supabase.com/docs/guides/database/orioledb", "title": "OrioleDB Overview" }, { - "url": "https://supabase.com/docs/guides/database/postgres/indexes", - "title": "Managing Indexes in Postgres" + "url": "https://supabase.com/docs/guides/api/rest/postgrest-error-codes", + "title": "Error Codes" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/timeouts", + "title": "Timeouts" } ], - "resultChars": 18373 + "resultChars": 35246 } ] }, @@ -10797,31 +10420,59 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"row level security policy auth.uid select using with check memberships supabase\", limit: 5) { nodes { __typename title href content } } }", + "query": "query { searchDocs(query: \"RLS policy EXISTS auth.uid org_id notes memberships\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" + }, { "url": "https://supabase.com/docs/guides/realtime/authorization", "title": "Realtime Authorization" }, { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" + "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", + "title": "Column Level Security" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" + "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", + "title": "Declarative database schemas" }, + { + "url": "https://supabase.com/docs/guides/auth/auth-anonymous", + "title": "Anonymous Sign-Ins" + } + ], + "resultChars": 55637 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"auth.uid request.jwt.claim.sub row level security policy test\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", "title": "RAG with Permissions" }, { - "url": "https://supabase.com/docs/guides/auth/third-party/firebase-auth", - "title": "Firebase Auth" + "url": "https://supabase.com/docs/guides/local-development/testing/overview", + "title": "Testing Overview" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-mfa", + "title": "Multi-Factor Authentication" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" } ], - "resultChars": 75855 + "resultChars": 63308 } ] }, @@ -10862,7 +10513,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user fbab494f-6a4b-4376-b26a-10e30c44a54d, signUp returned {\"userId\":\"fbab494f-6a4b-4376-b26a-10e30c44a54d\"}" + "notes": "db user 459847f0-1c32-4c34-9591-af8af3889fe5, signUp returned {\"userId\":\"459847f0-1c32-4c34-9591-af8af3889fe5\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -10877,7 +10528,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"fbab494f-6a4b-4376-b26a-10e30c44a54d\"}" + "notes": "{\"userId\":\"459847f0-1c32-4c34-9591-af8af3889fe5\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -11006,11 +10657,12 @@ }, { "name": "a new migration was generated for the change", - "passed": true + "passed": false, + "notes": "found 1 migration file(s)" }, { "name": "description column exists in the live database", - "passed": false + "passed": true } ], "skills": { @@ -11063,7 +10715,7 @@ { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 6) from the queue" + "notes": "function removed the seeded message (id 4) from the queue" } ], "skills": { @@ -11073,58 +10725,108 @@ "docs": { "calls": [ { - "source": "web_search", - "query": "Supabase pg_cron schedule cron.schedule syntax official docs", - "pages": [] + "source": "search_docs", + "query": "query {\n searchDocs(query: \"pgmq pop send delete queue Supabase Edge Function local\", limit: 5) {\n nodes {\n ... on Guide {\n title\n href\n content\n }\n ... on ClientLibraryFunctionReference {\n title\n href\n content\n language\n methodName\n }\n ... on TroubleshootingGuide {\n title\n href\n content\n }\n ... on CLICommandReference {\n title\n href\n content\n }\n ... on ManagementApiReference {\n title\n href\n content\n }\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "title": "Consuming Supabase Queue Messages with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" + }, + { + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" + }, + { + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/send-emails", + "title": "Sending Emails" + } + ], + "resultChars": 17810 }, { - "source": "web_search", - "query": "site:supabase.com/docs pgmq create queue function create queue", - "pages": [] + "source": "search_docs", + "query": "query {\n searchDocs(query: \"Supabase Cron schedule every minute edge function database cron job pg_cron\", limit: 5) {\n nodes {\n ... on Guide {\n title\n href\n content\n }\n ... on CLICommandReference {\n title\n href\n content\n }\n ... on TroubleshootingGuide {\n title\n href\n content\n }\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, + { + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-hooks/send-email-hook", + "title": "Send Email Hook" + }, + { + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" + } + ], + "resultChars": 95133 }, { - "source": "web_search", - "query": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "source": "search_docs", + "query": "query {\n searchDocs(query: \"pgmq create queue create('tasks') Supabase\", limit: 10) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n }\n }\n}", + "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions" + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + }, + { + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" + }, + { + "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues", + "title": "Expose Queues for local and self-hosted Supabase" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pgmq", + "title": "pgmq: Queues" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" + }, + { + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "title": "Consuming Supabase Queue Messages with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/42501--permission-denied-for-table-httprequestqueue-KnozmQ", + "title": "42501 : permission denied for table http_request_queue" } - ] - }, - { - "source": "web_search", - "query": "'pop(' in https://supabase.com/docs/guides/queues/pgmq", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs cron.schedule pg_cron supabase example", - "pages": [] - }, - { - "source": "web_search", - "query": "site:github.com supabase pg_cron cron.schedule supabase migration", - "pages": [] - }, - { - "source": "web_search", - "query": "'if not exists' in https://supabase.com/docs/guides/queues/pgmq", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs pg_cron unschedule cron.schedule idempotent", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs pgmq message_record msg_id message field", - "pages": [] - }, - { - "source": "web_search", - "query": "site:supabase.com/docs edge function config.toml verify_jwt false", - "pages": [] + ], + "resultChars": 128772 } ] }, @@ -11368,109 +11070,249 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Edge Functions verify_jwt auth getUser service role private notes\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"Edge Functions supabase-js getUser Authorization header anon key\", limit: 5) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", "title": "Integrating With Supabase Auth" }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#fetching-the-user", + "title": "Fetching the user" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#setting-up-auth-context", + "title": "Setting up auth context" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt#example", + "title": "Example" + }, + { + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions#when-to-use-edge-functions", + "title": "When to use Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions#quick-technical-notes", + "title": "Quick technical notes" + }, + { + "url": "https://supabase.com/docs/guides/functions#how-it-works", + "title": "How it works" + }, + { + "url": "https://supabase.com/docs/guides/functions#examples", + "title": "Examples" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs#profile-photos", + "title": "Profile photos" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs#create-an-upload-widget", + "title": "Create an upload widget" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs#update-the-account-form", + "title": "Update the account form" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs#launch", + "title": "Launch" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs#see-also", + "title": "See also" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs#project-setup", + "title": "Project setup" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs#create-a-project", + "title": "Create a project" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs#set-up-the-database-schema", + "title": "Set up the database schema" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs#get-api-details", + "title": "Get API details" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs#building-the-app", + "title": "Building the app" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs#initialize-a-nextjs-app", + "title": "Initialize a Next.js app" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs#app-styling-optional", + "title": "App styling (optional)" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs#supabase-server-side-auth-package", + "title": "Supabase Server-Side Auth package" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs#supabase-utilities", + "title": "Supabase utilities" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs#nextjs-proxy", + "title": "Next.js proxy" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs#summary-of-the-methods", + "title": "Summary of the methods" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs#login-and-signup-form", + "title": "Login and signup form" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs#set-up-a-login-page", + "title": "Set up a login page" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs#email-template", + "title": "Email template" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs#confirmation-endpoint", + "title": "Confirmation endpoint" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs#account-page", + "title": "Account page" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs#sign-out", + "title": "Sign out" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#combining-modes", + "title": "Combining modes" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#custom-error-responses", + "title": "Custom error responses" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#environment-variables", + "title": "Environment variables" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#authenticated-user-calls", + "title": "Authenticated user calls" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#service-to-service-calls", + "title": "Service-to-service calls" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#public-functions", + "title": "Public functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth#external-webhooks", + "title": "External webhooks" + }, { "url": "https://supabase.com/docs/guides/getting-started/api-keys", "title": "Understanding API keys" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/getting-started/api-keys#known-limitations-and-compatibility-differences", + "title": "Known limitations and compatibility differences" }, { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" + "url": "https://supabase.com/docs/guides/getting-started/api-keys#publishable-keys", + "title": "Publishable keys" }, { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - } - ], - "resultChars": 39139 - } - ] - }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-functions-004-service-role-bypass.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "build-functions-005-dual-auth-user-secret", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "sdk", - "rls", - "security" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 401: {\"error\":\"Unauthorized\"}" - }, - { - "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"a9708acd-ab70-4e53-8f37-66b0ed3c7434\",\"metric\":\"steps_a_msj11abm\",\"value\":111}]" - }, - { - "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: [{\"user_id\":\"a9708acd-ab70-4e53-8f37-66b0ed3c7434\",\"metric\":\"steps_a_msj11abm\",\"value\":111}]" - }, - { - "name": "service key bypasses RLS to read the target user's rows", - "passed": false, - "notes": "status 401: {\"error\":\"Unauthorized\"}" - }, - { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 401: {\"error\":\"Unauthorized\"}" - }, - { - "name": "rejects an unverified (forged) user token", - "passed": true, - "notes": "status 401: {\"error\":\"Unauthorized\"}" - }, - { - "name": "a user token in the apikey slot is not treated as the service key", - "passed": true, - "notes": "status 401: {\"error\":\"Unauthorized\"}" - }, + "url": "https://supabase.com/docs/guides/getting-started/api-keys#overview", + "title": "Overview" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys#what-to-do-if-a-secret-key-or-service_role-has-been-leaked-or-compromised", + "title": "What to do if a secret key or service_role has been leaked or compromised?" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys#best-practices-for-handling-secret-keys", + "title": "Best practices for handling secret keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys#what-secret-keys-allow-access-to", + "title": "What secret keys allow access to" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys#security-considerations", + "title": "Security considerations" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys#interaction-with-supabase-auth", + "title": "Interaction with Supabase Auth" + } + ], + "resultChars": 180034 + } + ] + }, + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-functions-004-service-role-bypass.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-functions-005-dual-auth-user-secret", + "stage": "build", + "product": [ + "edge-functions", + "auth", + "database" + ], + "topic": [ + "sdk", + "rls", + "security" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ { - "name": "implementation uses @supabase/server", + "name": "read stack config from `supabase status`", "passed": false, - "notes": "hand-rolled (raw supabase-js or other) — this eval requires @supabase/server" + "notes": "missing API_URL/SECRET_KEY/PUBLISHABLE_KEY — new API keys are required for @supabase/server; is the stack running on a new-enough CLI? got keys: ANON_KEY, DB_URL, JWT_SECRET, PUBLISHABLE_KEY, SECRET_KEY, SERVICE_ROLE_KEY" } ], "skills": { @@ -11481,31 +11323,59 @@ "calls": [ { "source": "search_docs", - "query": "query {\n searchDocs(query: \"edge function get user auth supabase service role apikey header SUPABASE_SERVICE_ROLE_KEY SUPABASE_ANON_KEY\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", + "query": "query {\n searchDocs(query: \"Edge Function authenticate user access token service_role apikey header supabase server createClient verify JWT\", limit: 5) {\n edges {\n node {\n title\n href\n content\n }\n }\n }\n}", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", "title": "Migrating to publishable and secret API keys" }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" + }, { "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", "title": "New API Keys and Asymmetric Authentication" + } + ], + "resultChars": 50589 + }, + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"supabase config.toml functions verify_jwt edge function config.toml\", limit: 5) {\n edges {\n node {\n title\n href\n content\n }\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/functions/deploy", + "title": "Deploy to Production" + }, + { + "url": "https://supabase.com/docs/guides/functions/development-tips", + "title": "Development tips" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", + "title": "Building an MCP Server with mcp-lite" + }, + { + "url": "https://supabase.com/docs/guides/ai-tools/byo-mcp", + "title": "Deploy MCP servers" } ], - "resultChars": 56285 + "resultChars": 31896 } ] }, @@ -11552,7 +11422,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019fdc8c-c70c-75ca-a770-9cd6c0d41bf9/receipt-alpha.pdf, 019fdc8c-c70c-75ca-a770-9cd6c0d41bf9/receipt-beta.pdf" + "notes": "saw: 01a0005d-d2b9-7728-9418-9abf1d2f9869/receipt-alpha.pdf, 01a0005d-d2b9-7728-9418-9abf1d2f9869/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -11573,7 +11443,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Meets rubric: private user-files bucket, RLS remains enabled, authenticated SELECT and INSERT policies scoped to bucket and auth.uid() path prefix, and supabase-js uses createSignedUrl with expiry." + "judgeNotes": "Meets all requirements: private user-files bucket, authenticated owner-scoped SELECT and INSERT policies on storage.objects with RLS left enabled, and supabase-js createSignedUrl with expiry for temporary sharing." } ], "skills": { @@ -11584,31 +11454,59 @@ "calls": [ { "source": "search_docs", - "query": "query {\n searchDocs(query: \"storage.foldername policy storage.objects auth.uid bucket private files\", limit: 5) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n ... on CLICommandReference { title href content }\n ... on ManagementApiReference { title href content }\n }\n }\n}", + "query": "query { searchDocs(query: \"Supabase Storage policies storage.objects signed URL private bucket createSignedUrl\", limit: 5) { nodes { __typename title href ... on Guide { content } ... on CLICommandReference { content } ... on ManagementApiReference { content } ... on ClientLibraryFunctionReference { content methodName language } ... on TroubleshootingGuide { content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" + "url": "https://supabase.com/docs/guides/storage/serving/downloads", + "title": "Serving assets from Storage" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/why-cant-i-uploadlistetc-my-public-bucket-Z6CmGt", - "title": "Why can't I upload/list/etc my public bucket?" + "url": "https://supabase.com/docs/guides/storage/s3/compatibility", + "title": "S3 Compatibility" + }, + { + "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", + "title": "Storage Image Transformations" }, { - "url": "https://supabase.com/docs/guides/storage/quickstart", - "title": "Storage Quickstart" + "url": "https://supabase.com/docs/guides/functions/examples/amazon-bedrock-image-generator", + "title": "Generate Images with Amazon Bedrock" }, { "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", "title": "Storage Buckets" + } + ], + "resultChars": 68767 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase Storage access control storage.objects policy foldername owner user id\", limit: 5) { nodes { __typename title href ... on Guide { content } ... on TroubleshootingGuide { content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/security/ownership", + "title": "Ownership" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/supabase-storage-inefficient-folder-operations-and-hierarchical-rls-challenges-b05a4d", + "title": "Supabase Storage: Inefficient folder operations and hierarchical RLS challenges" + }, + { + "url": "https://supabase.com/docs/guides/storage", + "title": "Storage" + }, + { + "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", + "title": "Custom Roles" } ], - "resultChars": 25446 + "resultChars": 15894 } ] }, @@ -11652,7 +11550,7 @@ { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as the table with the tenant isolation flaw, not `notes`, and treats pgTAP verification as the signal after adding isolation tests. It describes fixing `posts` RLS so authenticated users cannot read other organizations' posts." + "judgeNotes": "Correctly identifies `posts` as the table with the tenant isolation flaw, explains that its policy allowed users with any membership to read posts outside their org, and does not blame `notes` or dismiss pgTAP results." } ], "skills": { @@ -11662,10 +11560,208 @@ "docs": { "calls": [] }, - "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", - "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-tests-001-rls-tenant-isolation.json" + "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", + "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-tests-001-rls-tenant-isolation.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-vectors-001-rag-with-permissions", + "stage": "build", + "product": [ + "database", + "vectors" + ], + "topic": [ + "sql", + "rls" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "document_sections.embedding is vector(384)", + "passed": true, + "notes": "vector(384)" + }, + { + "name": "HNSW index on the embedding column", + "passed": true, + "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" + }, + { + "name": "index operator class matches the search operator", + "passed": true, + "notes": "function operators: <#>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" + }, + { + "name": "user A search returns only own sections, best match first", + "passed": true + }, + { + "name": "user B search returns only own sections, best match first", + "passed": true + }, + { + "name": "user A reads only own sections through the API", + "passed": true + }, + { + "name": "user A reads only own documents through the API", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"pgvector match_document_sections Supabase SQL function vector index row level security\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", + "title": "Semantic Search" + }, + { + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" + }, + { + "url": "https://supabase.com/docs/guides/ai/vector-columns", + "title": "Vector columns" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pgvector", + "title": "pgvector: Embeddings and vector similarity" + } + ], + "resultChars": 45063 + } + ] + }, + "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", + "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-vectors-001-rag-with-permissions.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "deploy-database-001-prometheus-metrics", + "stage": "deploy", + "product": [ + "database" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "passed": false, + "checks": [ + { + "name": "preserved existing app scrape job", + "passed": true + }, + { + "name": "configured the Supabase Metrics API scrape correctly", + "passed": false, + "judgeNotes": "Supabase scrape preserves the app job and uses HTTPS with the correct metrics path and project target pattern, but it uses basic_auth.password instead of basic_auth.password_file. docker-compose.yml also does not mount a password_file via a volume or Compose secret, so the required secret wiring is missing." + }, + { + "name": "documented live deployment and verification steps", + "passed": false, + "judgeNotes": "README explains creating/copying a Supabase Secret API key and recreating the Compose stack, but it uses environment variables/.env rather than placing a matching secret file as required. It also lacks concrete verification steps via Prometheus targets, PromQL, Grafana, or equivalent." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase metrics Prometheus project metrics endpoint\", limit: 10) { edges { node { title href content ... on Guide { subsections { nodes { title href content } } } } } totalCount } }", + "hasContent": true, + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"metrics prometheus\", limit: 10) { totalCount nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", + "title": "Metrics API" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" + }, + { + "url": "https://supabase.com/docs/reference/api/v1-scrape-project-metrics", + "title": "Scrape a project's metrics" + }, + { + "url": "https://supabase.com/docs/guides/database/replication/manual-replication-monitoring", + "title": "Manual replication monitoring" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/how-to-view-database-metrics-uqf2z_", + "title": "How to View Database Metrics" + }, + { + "url": "https://supabase.com/docs/guides/database/replication/pipelines-monitoring", + "title": "Monitor pipeline status" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/reports", + "title": "Reports" + } + ], + "resultChars": 89531 + } + ] + }, + "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", + "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-database-001-prometheus-metrics.json" }, { "experiment": "codex-gpt-5.4-mini-no-skills", @@ -11676,49 +11772,34 @@ "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "build-vectors-001-rag-with-permissions", - "stage": "build", + "eval": "deploy-functions-001-edge-function-secrets", + "stage": "deploy", "product": [ - "database", - "vectors" + "edge-functions" ], "topic": [ - "sql", - "rls" + "security" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", "passed": true, "checks": [ { - "name": "document_sections.embedding is vector(384)", - "passed": true, - "notes": "vector(384)" + "name": "WEATHER_API_KEY is set as a Function secret on the project", + "passed": true }, { - "name": "HNSW index on the embedding column", + "name": "the weather function is deployed to the project", "passed": true, - "notes": "CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" + "notes": "status ACTIVE" }, { - "name": "index operator class matches the search operator", + "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops)" - }, - { - "name": "user A search returns only own sections, best match first", - "passed": true - }, - { - "name": "user B search returns only own sections, best match first", - "passed": true - }, - { - "name": "user A reads only own sections through the API", - "passed": true + "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." }, { - "name": "user A reads only own documents through the API", + "name": "WEATHER_API_KEY value is not committed to the repo", "passed": true } ], @@ -11729,21 +11810,59 @@ "docs": { "calls": [ { - "source": "web_search", - "query": "site:supabase.com gte-small 384 Supabase AI Session dimension", - "pages": [] + "source": "search_docs", + "query": "query {\n searchDocs(query: \"Supabase Edge Function deploy secrets env-file set --env-file runtime environment variable WEATHER_API_KEY\", limit: 5) {\n nodes {\n ... on Guide { title href content }\n ... on CLICommandReference { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n ... on ManagementApiReference { title href content }\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", + "title": "Transcription Telegram Bot" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", + "title": "Streaming Speech with ElevenLabs" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/ai/examples/openai", + "title": "Generating OpenAI GPT3 completions" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + } + ], + "resultChars": 42901 }, { - "source": "web_search", - "query": "site:supabase.com/docs pgvector Supabase vector search match function", - "pages": [] + "source": "search_docs", + "query": "query {\n searchDocs(query: \"supabase config.toml functions verify_jwt false weather\", limit: 3) {\n nodes {\n ... on Guide { title href content }\n ... on CLICommandReference { title href content }\n ... on TroubleshootingGuide { title href content }\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/development-tips", + "title": "Development tips" + }, + { + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" + }, + { + "url": "https://supabase.com/docs/guides/functions/deploy", + "title": "Deploy to Production" + } + ], + "resultChars": 15421 } ] }, - "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", - "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", + "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", + "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-vectors-001-rag-with-permissions.json" + "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-functions-001-edge-function-secrets.json" }, { "experiment": "codex-gpt-5.4-mini-no-skills", @@ -11754,30 +11873,35 @@ "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "deploy-database-001-prometheus-metrics", + "eval": "deploy-self-hosting-001-docker-compose", "stage": "deploy", "product": [ - "database" + "database", + "auth", + "storage" ], "topic": [ - "observability" + "self-hosting" ], "suite": "benchmark", - "passed": false, + "interface": "cli", + "passed": true, "checks": [ { - "name": "preserved existing app scrape job", + "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", "passed": true }, { - "name": "configured the Supabase Metrics API scrape correctly", - "passed": false, - "judgeNotes": "Fails because Prometheus uses basic_auth.password with an environment variable instead of basic_auth.password_file, and docker-compose.yml does not mount the password file via a volume or Compose secret. App scrape and endpoint are otherwise preserved." + "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", + "passed": true }, { - "name": "documented live deployment and verification steps", - "passed": false, - "judgeNotes": "README includes env setup, restart/recreate Compose, and Prometheus target verification, but it does not provide steps to create the Secret API key in Supabase or place a matching secret file. The setup uses .env rather than the required secret file, so the secret setup requirement is not met." + "name": "secrets rotated off the shipped defaults", + "passed": true + }, + { + "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", + "passed": true } ], "skills": { @@ -11788,507 +11912,765 @@ "calls": [ { "source": "search_docs", - "query": "query {\n searchDocs(query: \"metrics prometheus project metrics scrape endpoint\", limit: 10) {\n nodes {\n ... on Guide {\n title\n href\n content\n }\n ... on TroubleshootingGuide {\n title\n href\n content\n }\n ... on ClientLibraryFunctionReference {\n title\n href\n content\n language\n methodName\n }\n }\n totalCount\n }\n}", + "query": "query { searchDocs(query: \"self-hosting docker compose .env secrets JWT_SECRET DASHBOARD_USERNAME POSTGRES_PASSWORD Supabase Docker\", limit: 5) { nodes { __typename ... on Guide { title href content subsections { nodes { title href content } } } ... on CLICommandReference { title href content } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-secrets", + "title": "Configuring secrets" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-edge-functions", + "title": "Accessing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-apis", + "title": "Accessing APIs" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#enabling-analytics", + "title": "Enabling analytics" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-https", + "title": "Configuring HTTPS" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#managing-the-stack", + "title": "Managing the stack" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#updating", + "title": "Updating" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#uninstalling", + "title": "Uninstalling" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#advanced-topics", + "title": "Advanced topics" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#architecture", + "title": "Architecture" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#setting-database-password", + "title": "Setting database password" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#changing-database-password", + "title": "Changing database password" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-supabase-services", + "title": "Configuring Supabase services" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-social-login-oauth-providers", + "title": "Configuring social login (OAuth) providers" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-phone-login-sms-and-mfa", + "title": "Configuring phone login, SMS, and MFA" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-an-email-server", + "title": "Configuring an email server" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-s3-storage", + "title": "Configuring S3 Storage" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#using-file-backend-in-storage-on-macos", + "title": "Using file backend in Storage on macOS" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-supabase-ai-assistant", + "title": "Configuring Supabase AI Assistant" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-postgres-through-supavisor", + "title": "Accessing Postgres through Supavisor" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#exposing-your-postgres-database", + "title": "Exposing your Postgres database" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#setting-log_min_messages-in-postgres", + "title": "Setting log_min_messages in Postgres" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#managing-your-secrets", + "title": "Managing your secrets" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#demo", + "title": "Demo" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#contents", + "title": "Contents" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" + "url": "https://supabase.com/docs/guides/self-hosting/docker#before-you-begin", + "title": "Before you begin" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" + "url": "https://supabase.com/docs/guides/self-hosting/docker#system-requirements", + "title": "System requirements" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" + "url": "https://supabase.com/docs/guides/self-hosting/docker#installing-supabase", + "title": "Installing Supabase" }, { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" + "url": "https://supabase.com/docs/guides/self-hosting/docker#quick-start-linux", + "title": "Quick start (Linux)" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/how-to-view-database-metrics-uqf2z_", - "title": "How to View Database Metrics" - } - ], - "resultChars": 29978 - } - ] - }, - "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", - "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-database-001-prometheus-metrics.json" - }, - { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.4-mini", - "reasoningEffort": "medium" - }, - "eval": "deploy-functions-001-edge-function-secrets", - "stage": "deploy", - "product": [ - "edge-functions" - ], - "topic": [ - "security" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "WEATHER_API_KEY is set as a Function secret on the project", - "passed": true - }, - { - "name": "the weather function is deployed to the project", - "passed": true, - "notes": "status ACTIVE" - }, - { - "name": "the weather function reads WEATHER_API_KEY from the environment", - "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." - }, - { - "name": "WEATHER_API_KEY value is not committed to the repo", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query:\"Edge Function environment variable deploy secrets WEATHER_API_KEY supabase functions secrets set\", limit: 5) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/self-hosting/docker#manual-installation", + "title": "Manual installation" + }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-and-securing-supabase", + "title": "Configuring and securing Supabase" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-1-create-the-new-api-keys", - "title": "Step 1: Create the new API keys" + "url": "https://supabase.com/docs/guides/self-hosting/docker#generate-keys-and-secrets", + "title": "Generate keys and secrets" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#before-you-start", - "title": "Before you start" + "url": "https://supabase.com/docs/guides/self-hosting/docker#configure-supabase-urls", + "title": "Configure Supabase URLs" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#next-steps", - "title": "Next steps" + "url": "https://supabase.com/docs/guides/self-hosting/docker#where-to-find-your-credentials", + "title": "Where to find your credentials" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#known-limitations", - "title": "Known limitations" + "url": "https://supabase.com/docs/guides/self-hosting/docker#studio-authentication", + "title": "Studio authentication" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-6-deactivate-the-legacy-keys", - "title": "Step 6: Deactivate the legacy keys" + "url": "https://supabase.com/docs/guides/self-hosting/docker#starting-and-stopping", + "title": "Starting and stopping" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-5-verify-nothing-uses-the-legacy-keys", - "title": "Step 5: Verify nothing uses the legacy keys" + "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-supabase-studio-dashboard", + "title": "Accessing Supabase Studio (Dashboard)" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#option-2-adopt-the-supabaseserver-sdk", - "title": "Option 2: Adopt the @supabase/server SDK" + "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-postgres", + "title": "Accessing Postgres" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#option-1-read-the-new-keys-from-the-environment", - "title": "Option 1: Read the new keys from the environment" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-4-update-edge-functions", - "title": "Step 4: Update Edge Functions" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#adding-the-new-keys", + "title": "Adding the new keys" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-3-swap-the-secret-key-in-backend-code", - "title": "Step 3: Swap the secret key in backend code" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#new-api-keys-format", + "title": "New API keys format" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#database-webhooks-and-pg_net", - "title": "Database Webhooks and pg_net" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#verifying-the-setup", + "title": "Verifying the setup" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys#step-2-swap-the-publishable-key-in-client-code", - "title": "Step 2: Swap the publishable key in client code" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#environment-variables-configuration", + "title": "Environment variables configuration" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", - "title": "Transcription Telegram Bot" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#differences-from-the-supabase-platform", + "title": "Differences from the Supabase platform" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#deploy-to-supabase", - "title": "Deploy to Supabase" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#backward-compatibility", + "title": "Backward compatibility" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#introduction", - "title": "Introduction" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#rotating-the-new-api-keys", + "title": "Rotating the new API keys" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#requirements", - "title": "Requirements" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#regenerating-asymmetric-key-pair", + "title": "Regenerating asymmetric key pair" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#how-it-works", + "title": "How it works" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#setup", - "title": "Setup" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#what-client-sdk-sends", + "title": "What client SDK sends" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#register-a-telegram-bot", - "title": "Register a Telegram bot" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#kong-api-gateway-routing", + "title": "Kong API gateway routing" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#create-a-supabase-project-locally", - "title": "Create a Supabase project locally" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#request-flows", + "title": "Request flows" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#create-a-database-table-to-log-the-transcription-results", - "title": "Create a database table to log the transcription results" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#unauthenticated-requests-api-key-only-no-user-session-jwt", + "title": "Unauthenticated requests (API key only, no user session JWT)" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#create-a-supabase-edge-function-to-handle-telegram-webhook-requests", - "title": "Create a Supabase Edge Function to handle Telegram webhook requests" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#authenticated-requests-user-session-jwt", + "title": "Authenticated requests (user session JWT)" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#set-up-the-environment-variables", - "title": "Set up the environment variables" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#additional-resources", + "title": "Additional resources" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#dependencies", - "title": "Dependencies" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#before-you-begin", + "title": "Before you begin" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#code-the-telegram-bot", - "title": "Code the Telegram bot" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17", + "title": "Upgrade to Postgres 17" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#apply-the-database-migrations", - "title": "Apply the database migrations" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#pgsodium--supabase-vault-errors", + "title": "pgsodium / Supabase Vault errors" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#set-up-the-webhook", - "title": "Set up the webhook" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#services-fail-to-connect-after-upgrade", + "title": "Services fail to connect after upgrade" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#set-the-function-secrets", - "title": "Set the function secrets" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#disk-space-issues-during-upgrade", + "title": "Disk space issues during upgrade" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech#test-the-bot", - "title": "Test the bot" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#restoring-from-a-manual-backup", + "title": "Restoring from a manual backup" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", - "title": "Streaming Speech with ElevenLabs" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#postgres-17-fails-to-start-with-a-leftover-db-config-volume", + "title": "Postgres 17 fails to start with a leftover db-config volume" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#deploy-to-supabase", - "title": "Deploy to Supabase" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#after-the-upgrade", + "title": "After the upgrade" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#try-it-out", - "title": "Try it out" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#run-the-upgrade", + "title": "Run the upgrade" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#run-locally", - "title": "Run locally" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#extensions-removed-in-postgres-17", + "title": "Extensions removed in Postgres 17" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#code-the-supabase-edge-function", - "title": "Code the Supabase Edge Function" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#requirements", + "title": "Requirements" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#dependencies", - "title": "Dependencies" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#upgrade-an-existing-postgres-15-deployment", + "title": "Upgrade an existing Postgres 15 deployment" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#set-up-the-environment-variables", - "title": "Set up the environment variables" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#new-deployment-with-postgres-17", + "title": "New deployment with Postgres 17" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#create-a-supabase-edge-function-for-speech-generation", - "title": "Create a Supabase Edge Function for speech generation" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#before-you-begin", + "title": "Before you begin" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#configure-background-tasks-for-supabase-edge-functions", - "title": "Configure background tasks for Supabase Edge Functions" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#create-a-backup", + "title": "Create a backup" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#configure-the-storage-bucket", - "title": "Configure the storage bucket" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#what-the-upgrade-does", + "title": "What the upgrade does" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#create-a-supabase-project-locally", - "title": "Create a Supabase project locally" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#rollback", + "title": "Rollback" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#setup", - "title": "Setup" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#custom-postgres-configuration", + "title": "Custom Postgres configuration" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#introduction", - "title": "Introduction" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#upgrade-process-details", + "title": "Upgrade process details" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#requirements", - "title": "Requirements" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#troubleshooting", + "title": "Troubleshooting" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#test-the-function", - "title": "Test the function" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#pg_upgrade-fails-with-replication-slot-errors", + "title": "pg_upgrade fails with replication slot errors" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream#set-the-function-secrets", - "title": "Set the function secrets" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" }, { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#connection-refused", + "title": "Connection refused" }, { - "url": "https://supabase.com/docs/guides/functions/secrets#accessing-environment-variables", - "title": "Accessing environment variables" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#legacy-studio-configuration", + "title": "Legacy Studio configuration" }, { - "url": "https://supabase.com/docs/guides/functions/secrets#local-secrets", - "title": "Local secrets" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#custom-roles-missing-passwords", + "title": "Custom roles missing passwords" }, { - "url": "https://supabase.com/docs/guides/functions/secrets#production-secrets", - "title": "Production secrets" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#additional-resources", + "title": "Additional resources" }, { - "url": "https://supabase.com/docs/guides/functions/secrets#default-secrets", - "title": "Default secrets" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#before-you-begin", + "title": "Before you begin" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#step-1-get-your-platform-connection-string", + "title": "Step 1: Get your platform connection string" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#changing-compute-sizes", - "title": "Changing compute sizes" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#step-2-back-up-your-platform-database", + "title": "Step 2: Back up your platform database" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#recommended-api-keys", - "title": "Recommended API keys" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#step-3-prepare-your-self-hosted-instance", + "title": "Step 3: Prepare your self-hosted instance" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#nano-compute-instance", - "title": "Nano compute instance" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#step-4-restore-to-your-self-hosted-database", + "title": "Step 4: Restore to your self-hosted database" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#launching-projects", - "title": "Launching projects" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#step-5-verify-the-restore", + "title": "Step 5: Verify the restore" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#overview", - "title": "Overview" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#whats-included-in-the-restore-and-whats-not", + "title": "What's included in the restore and what's not" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#auth-considerations", + "title": "Auth considerations" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#postgres-version-compatibility", + "title": "Postgres version compatibility" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#troubleshooting", + "title": "Troubleshooting" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#version-mismatches-between-platform-and-self-hosted", + "title": "Version mismatches between platform and self-hosted" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#extension-not-available", + "title": "Extension not available" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#platform-kit", - "title": "Platform kit" + "url": "https://supabase.com/docs/guides/self-hosting", + "title": "Self-Hosting" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#claim-flow", - "title": "Claim flow" + "url": "https://supabase.com/docs/guides/self-hosting#how-self-hosted-supabase-differs", + "title": "How self-hosted Supabase differs" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#disaster-recovery-for-production", - "title": "Disaster recovery for production" + "url": "https://supabase.com/docs/guides/self-hosting#enterprise-self-hosting", + "title": "Enterprise self-hosting" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#security-checks-for-production", - "title": "Security checks for production" + "url": "https://supabase.com/docs/guides/self-hosting#support-and-community", + "title": "Support and community" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#merge-all-changes", - "title": "Merge all changes" + "url": "https://supabase.com/docs/guides/self-hosting#get-started", + "title": "Get started" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#deploying-edge-functions", - "title": "Deploying Edge Functions" + "url": "https://supabase.com/docs/guides/self-hosting#telemetry", + "title": "Telemetry" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#add-seed-data", - "title": "Add seed data" + "url": "https://supabase.com/docs/guides/self-hosting#your-responsibilities-when-self-hosting", + "title": "Your responsibilities when self-hosting" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#reverting-changes", - "title": "Reverting changes" + "url": "https://supabase.com/docs/guides/self-hosting#community-driven-projects", + "title": "Community-driven projects" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#create-a-restore-point", - "title": "Create a restore point" + "url": "https://supabase.com/docs/guides/self-hosting#about-self-hosting", + "title": "About self-hosting" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#make-database-changes", - "title": "Make database changes" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#permission-denied-on-the-data-directory" + } + ], + "resultChars": 287144 + } + ] + }, + "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", + "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-self-hosting-001-docker-compose.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "investigate-auth-001-deleted-user-access", + "stage": "investigate", + "product": [ + "auth" + ], + "topic": [ + "security", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "victim session active before delete-account", + "passed": true + }, + { + "name": "delete_account flow ran for the victim", + "passed": true + }, + { + "name": "delete-account revokes the user's sessions", + "passed": true, + "notes": "sessions left: 0" + }, + { + "name": "deleted user's refresh token is rejected", + "passed": true + }, + { + "name": "deleted user cannot sign back in", + "passed": true + }, + { + "name": "other users keep their sessions and access", + "passed": true + }, + { + "name": "diagnosed and explained session revocation", + "passed": true, + "judgeNotes": "The answer identifies the soft-delete-only bug, implements auth.users deletion with cascading sessions/refresh token removal, and explains stale JWT caveat consistently: no Data API/RLS window due to auth user deletion/cascade, but local JWT validation remains valid until exp. It also correctly distinguishes publishable/anon vs secret/service_role and says secrets bypass RLS and must not be frontend." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"delete user revoke sessions auth users Supabase\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#creating-a-dev-branch", - "title": "Creating a DEV branch" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#development-workflow", - "title": "Development workflow" + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", + "title": "OAuth 2.1 Flows" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#configuration-changes", - "title": "Configuration changes" + "url": "https://supabase.com/docs/guides/platform/sso", + "title": "Enable SSO for Your Organization" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms#debugging-projects", - "title": "Debugging projects" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" } ], - "resultChars": 164975 + "resultChars": 98509 }, { "source": "search_docs", - "query": "query { searchDocs(query:\"Edge Functions invoke URL functions/v1 project ref invoke runtime endpoint\", limit: 10) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"publishable secret keys anon service_role RLS frontend backend\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/recursive-functions", - "title": "Recursive / Nested Function Calls" + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/auth/jwt-fields", + "title": "JWT Claims Reference" }, { - "url": "https://supabase.com/docs/guides/functions/storage-caching", - "title": "Integrating with Supabase Storage" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" }, { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + } + ], + "resultChars": 91173 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"auth.users deleted_at deleted user sign in after delete Supabase\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/functions/ai-models", - "title": "Running AI Models" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/troubleshooting/dashboard-errors-when-managing-users-N1ls4A", + "title": "Errors when creating / updating / deleting users" }, { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" + "url": "https://supabase.com/docs/guides/auth/auth-anonymous", + "title": "Anonymous Sign-Ins" }, { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" } ], - "resultChars": 93047 + "resultChars": 75709 }, { "source": "search_docs", - "query": "query { searchDocs(query:\"Management API edge functions get function by slug project ref functions/{function_slug}\", limit: 10) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"delete account sign out user sessions delete auth.sessions Supabase\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/api/v1-get-a-function", - "title": "Retrieve a function" - }, - { - "url": "https://supabase.com/docs/reference/api/v1-delete-a-function", - "title": "Delete a function" - }, - { - "url": "https://supabase.com/docs/reference/api/v1-update-a-function", - "title": "Update a function" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/reference/api/v1-list-all-functions", - "title": "List all functions" + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" }, { - "url": "https://supabase.com/docs/reference/api/v1-get-a-function-body", - "title": "Retrieve a function body" + "url": "https://supabase.com/docs/guides/auth/signout", + "title": "Signing out" }, { - "url": "https://supabase.com/docs/reference/api/v1-create-a-function", - "title": "Create a function" + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" }, { - "url": "https://supabase.com/docs/reference/api/v1-deploy-a-function", - "title": "Deploy a function" - }, + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" + } + ], + "resultChars": 41206 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"delete users auth.users does not automatically sign out jwt valid until expired Supabase\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/reference/api/v1-bulk-update-functions", - "title": "Bulk update functions" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/reference/api/v1-get-project-function-combined-stats", - "title": "Gets a project's function combined statistics" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/guides/ai-tools/mcp", - "title": "Supabase MCP Server" + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" } ], - "resultChars": 18063 + "resultChars": 52406 }, { "source": "search_docs", - "query": "query { searchDocs(query:\"get_project_url project url supabase management api\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"banned_until auth.users Supabase Auth banned user\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/analytics/connecting-to-analytics-bucket", - "title": "Iceberg Catalog" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/ai-tools/mcp", - "title": "Supabase MCP Server" + "url": "https://supabase.com/docs/guides/auth/users", + "title": "Users" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" }, { - "url": "https://supabase.com/docs/guides/api/creating-routes", - "title": "Creating API Routes" + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" } ], - "resultChars": 69182 + "resultChars": 49070 } ] }, - "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", - "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-functions-001-edge-function-secrets.json" + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-auth-001-deleted-user-access.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "investigate-realtime-001-subscribed-no-events", + "stage": "investigate", + "product": [ + "realtime", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "orders table added to supabase_realtime publication", + "passed": true + }, + { + "name": "courier_locations still in supabase_realtime publication", + "passed": true + }, + { + "name": "publication still publishes INSERT events", + "passed": true + }, + { + "name": "RLS still enabled on orders", + "passed": true + }, + { + "name": "staff can still read orders through RLS", + "passed": true, + "notes": "authenticated sees 2 of 2 orders" + }, + { + "name": "diagnosed missing publication membership", + "passed": true, + "judgeNotes": "Diagnosed orders missing from supabase_realtime publication, applied ALTER PUBLICATION to add public.orders, verified courier_locations remained and RLS/policies were not weakened." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", + "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-realtime-001-subscribed-no-events.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "investigate-reliability-003-edge-function-5xx-correlation", + "stage": "investigate", + "product": [ + "edge-functions" + ], + "topic": [ + "observability" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": false, + "checks": [ + { + "name": "identified image-transform and the recurring 503 pattern", + "passed": true, + "judgeNotes": "Identified image-transform as affected and described repeated HTTP 503 gateway responses throughout the morning of 2026-04-28, covering the failures from 07:00Z through 12:00Z with successful responses interleaved." + }, + { + "name": "attributed recurring 503s to gateway/platform layer, not function code", + "passed": false, + "judgeNotes": "The assistant does attribute the 503s partly to gateway/platform availability and cites valid observations like gateway-level 503s interleaved with 200s on the same deployment. However, it also treats deployment/runtime/dependency issues as possible causes and recommends redeploying or rolling back the functions, which the rubric explicitly marks as a failure." + }, + { + "name": "recommended a concrete next step", + "passed": true, + "judgeNotes": "The assistant recommended concrete next steps including checking dependency/deployment changes, pulling runtime stderr/stdout for failing timestamps, redeploying or rolling back functions, and inspecting deployment history." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", + "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-reliability-003-edge-function-5xx-correlation.json" }, { "experiment": "codex-gpt-5.4-mini-no-skills", @@ -12299,35 +12681,108 @@ "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", + "eval": "resolve-dataapi-001-empty-results", + "stage": "resolve", "product": [ + "data-api", "database", - "auth", - "storage" + "auth" ], "topic": [ - "self-hosting" + "rls", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS still enabled on bookmarks", + "passed": true + }, + { + "name": "user A reads own bookmarks", + "passed": true + }, + { + "name": "user B cannot read user A bookmarks", + "passed": true + }, + { + "name": "anon reads no bookmarks", + "passed": true + }, + { + "name": "user A can save a new bookmark", + "passed": true + }, + { + "name": "user B cannot insert a bookmark as user A", + "passed": true + }, + { + "name": "diagnosed RLS and added owner-scoped policies", + "passed": true, + "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies as deny-all, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() using USING and WITH CHECK." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", + "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-dataapi-001-empty-results.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "resolve-database-001-migration-history-mismatch", + "stage": "resolve", + "product": [ + "database" + ], + "topic": [ + "migrations" ], "suite": "benchmark", "interface": "cli", "passed": true, "checks": [ { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", + "name": "the avatar_url column is applied on the hosted profiles table", "passed": true }, { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", + "name": "migration 20240220000000 is recorded in the remote history", "passed": true }, { - "name": "secrets rotated off the shipped defaults", + "name": "remote migration history matches local migration files", "passed": true }, { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", + "name": "local migrations are a valid reconciled sequence", + "passed": true + }, + { + "name": "production profile data is intact (not reset)", "passed": true + }, + { + "name": "the avatar migration and history reconciliation were done via the Supabase CLI", + "passed": true, + "judgeNotes": "Avatar migration was applied through the Supabase CLI with `supabase db push --db-url ...` in action #25, whose output shows `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration was reconciled by adding the missing local migration file `supabase/migrations/20240115000000_add_profile_bio.sql` (#23), after which the same `supabase db push` succeeded. No disallowed workaround or direct mutation was used; psql commands were read-only inspections." } ], "skills": { @@ -12335,475 +12790,963 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "web_search", - "query": "Supabase self-hosting docker compose official secrets env file", - "pages": [] - }, - { - "source": "web_search", - "query": "site:github.com/supabase/supabase docker generate-keys.sh .env.example", - "pages": [] - } - ] + "calls": [] }, - "prompt": "I'm moving off the hosted Supabase and running the whole thing myself on a VPS I\njust spun up. Can you get a Docker setup ready for me to copy onto the box?\n\nI don't need it running here, I'll do the actual bring-up once I'm on the\nserver. I just want everything in place and the secrets set up. Put it in a `supabase-docker/`\nfolder at the repo root so I can scp the whole thing across in one go.", - "promptSourcePath": "evals/deploy-self-hosting-001-docker-compose/PROMPT.md", + "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", + "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/deploy-self-hosting-001-docker-compose.json" + "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-database-001-migration-history-mismatch.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "resolve-performance-001-slow-query-cpu-spike", + "stage": "resolve", + "product": [ + "database" + ], + "topic": [ + "observability", + "sql" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "inspected pg_stat_statements for query performance", + "passed": true + }, + { + "name": "ran EXPLAIN on the expensive query", + "passed": true + }, + { + "name": "created index covering user_id and created_at", + "passed": true + }, + { + "name": "query plan uses an index and avoids sequential scan", + "passed": true, + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_desc_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + }, + { + "name": "inserts still work", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", + "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-performance-001-slow-query-cpu-spike.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "resolve-security-002-rls-cross-tenant-leak", + "stage": "resolve", + "product": [ + "database", + "auth" + ], + "topic": [ + "rls", + "security" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS enabled on notes", + "passed": true + }, + { + "name": "tenant A sees only org A notes", + "passed": true + }, + { + "name": "tenant B cannot read org A notes", + "passed": true + }, + { + "name": "tenant A author can update own note", + "passed": true + }, + { + "name": "tenant B cannot update org A note", + "passed": true + }, + { + "name": "tenant B author can delete own note", + "passed": true + }, + { + "name": "tenant B cannot delete org A note", + "passed": true + }, + { + "name": "tenant A can insert note in own org", + "passed": true + }, + { + "name": "tenant B cannot insert into org A", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", + "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-security-002-rls-cross-tenant-leak.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "investigate-auth-001-deleted-user-access", - "stage": "investigate", + "eval": "build-auth-001-email-password-flow", + "stage": "build", "product": [ - "auth" + "auth", + "database" ], "topic": [ - "security", - "sdk" + "sdk", + "rls" ], "suite": "benchmark", - "interface": "mcp", - "passed": false, + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, "checks": [ { - "name": "victim session active before delete-account", - "passed": true + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" }, { - "name": "delete_account flow ran for the victim", - "passed": true + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user 6a747776-2287-4fa9-965c-9a63be5ace62, signUp returned {\"userId\":\"6a747776-2287-4fa9-965c-9a63be5ace62\"}" }, { - "name": "delete-account revokes the user's sessions", + "name": "signup metadata reaches the profile (display name)", "passed": true, - "notes": "sessions left: 0" + "notes": "profiles.display_name = \"Alex Doe\"" }, { - "name": "deleted user's refresh token is rejected", - "passed": true + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" }, { - "name": "deleted user cannot sign back in", - "passed": true + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"6a747776-2287-4fa9-965c-9a63be5ace62\"}" }, { - "name": "other users keep their sessions and access", - "passed": true + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" }, { - "name": "diagnosed and explained session revocation", - "passed": false, - "judgeNotes": "The answer correctly diagnoses the soft-delete bug, deletes the auth user so sessions/refresh tokens cascade, adds RLS that blocks stale JWTs on the data path, and correctly explains publishable vs secret keys. However, it does not clearly state the required caveat that access tokens are stateless JWTs not recalled by deletion/revocation and that purely local validation such as getClaims/custom JWT middleware will continue accepting them until expiry." + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [ { - "source": "web_search", - "query": "site:supabase.com/docs publishable secret keys supabase frontend secret keys rls", - "pages": [] + "source": "search_docs", + "query": "query { searchDocs(query: \"supabase-js auth signUp email password options data signInWithPassword getUser select single profile\", limit: 8) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", + "title": "Customizing Emails by Language" + }, + { + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-refine", + "title": "Build a User Management App with Refine" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" + }, + { + "url": "https://supabase.com/docs/guides/auth/passkeys", + "title": "Passkey authentication" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", + "title": "Passwordless email logins" + } + ], + "resultChars": 150758 }, { - "source": "web_search", - "query": "https://supabase.com/docs/guides/getting-started/api-keys", + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- changelog auth-related ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|auth|signUp|signInWithPassword' | head -n 160; printf '%s\\\\n' '--- files ---'; rg --files -g 'AGENTS.md' -g '\"'!node_modules'\"' -g '\"'!supabase/.temp'\"' .; printf '%s\\\\n' '--- app/auth ---'; sed -n '1,240p' app/src/auth.mjs; printf '%s\\\\n' '--- app tree/package ---'; find app -maxdepth 3 -type f -not -path '*/node_modules/*' -print | sort; cat app/package.json; printf '%s\\\\n' '--- supabase files ---'; find supabase -maxdepth 4 -type f -not -path '*/.temp/*' -print | sort; printf '%s\\\\n' '--- migrations/schema content ---'; for f in \"'$(find supabase -maxdepth 4 -type f '\"\\\\( -name '*.sql' -o -name 'config.toml' \\\\) -not -path '*/.temp/*' | sort); do echo \\\"### \"'$f\"; sed -n '\"'1,260p' \\\"\"'$f\"; done'", "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/api-keys" + "url": "https://supabase.com/changelog.md" } - ] + ], + "resultChars": 16692 }, { - "source": "web_search", - "query": "https://supabase.com/docs/guides/auth/jwts", + "source": "search_docs", + "query": "query { searchDocs(query: \"JavaScript signUp signInWithPassword getUser select maybeSingle createClient auth reference\", limit: 10) { nodes { title href content } } }", + "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/jwts" + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" + }, + { + "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", + "title": "signInWithPassword()" + }, + { + "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" + }, + { + "url": "https://supabase.com/docs/reference/csharp/auth-signinwithpassword", + "title": "SignIn(email, password)" + }, + { + "url": "https://supabase.com/docs/reference/kotlin/auth-signinwithpassword", + "title": "loginWith()" + }, + { + "url": "https://supabase.com/docs/reference/swift/auth-signup", + "title": "signUp()" + }, + { + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" + }, + { + "url": "https://supabase.com/docs/reference/kotlin/auth-signup", + "title": "signUp()" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpasskey" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithoauth" } - ] + ], + "resultChars": 55559 } ] }, - "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", - "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-auth-001-deleted-user-access.json" + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6/build-auth-001-email-password-flow.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "investigate-realtime-001-subscribed-no-events", - "stage": "investigate", + "eval": "build-cli-001-bootstrap-app", + "stage": "build", "product": [ - "realtime", - "database" + "database", + "data-api" ], "topic": [ - "sdk" + "migrations", + "rls" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", "passed": true, "checks": [ { - "name": "orders table added to supabase_realtime publication", + "name": "supabase project initialised (supabase/config.toml exists)", "passed": true }, { - "name": "courier_locations still in supabase_realtime publication", + "name": "todos table is created by a migration file", "passed": true }, { - "name": "publication still publishes INSERT events", + "name": "todos table exists with at least 2 seeded rows", + "passed": true, + "notes": "found 2 rows" + }, + { + "name": "row level security is enabled on todos", "passed": true }, { - "name": "RLS still enabled on orders", + "name": "a SELECT policy targets the authenticated role", "passed": true }, { - "name": "staff can still read orders through RLS", + "name": "REST API returns no todos to anonymous requests", "passed": true, - "notes": "authenticated sees 2 of 2 orders" + "notes": "0 rows" }, { - "name": "diagnosed missing publication membership", + "name": "REST API returns the todos to authenticated requests", "passed": true, - "judgeNotes": "The assistant correctly identified that orders was missing from the supabase_realtime publication despite the channel subscribing, added only public.orders to the existing publication, and preserved courier_locations, RLS, and policies." + "notes": "2 rows" } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- skill continuation ---'; sed -n '1,260p' .claude/skills/supabase/SKILL.md; printf '%s\\\\n' '--- relevant references ---'; find .claude/skills/supabase-postgres-best-practices/references -maxdepth 1 -type f | sort | rg '(security|schema)' | head -30; printf '%s\\\\n' '--- changelog breaking tags ---'; curl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|breaking change' | head -30; printf '%s\\\\n' '--- cli version/status ---'; supabase --version; git status --short; ls -la\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 3037 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"local development migrations seed.sql row level security authenticated role select policy Data API grants\", limit: 8) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/roles", + "title": "Postgres Roles" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" + } + ], + "resultChars": 118409 + } + ] }, - "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", - "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", + "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", + "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-realtime-001-subscribed-no-events.json" + "sourcePath": "codex-gpt-5.6/build-cli-001-bootstrap-app.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "investigate-reliability-003-edge-function-5xx-correlation", - "stage": "investigate", + "eval": "build-cli-002-declarative-schema", + "stage": "build", "product": [ - "edge-functions" + "database" ], "topic": [ - "observability" + "declarative-schema", + "migrations" ], "suite": "benchmark", - "interface": "mcp", - "passed": false, + "interface": "cli", + "passed": true, "checks": [ { - "name": "identified image-transform and the recurring 503 pattern", - "passed": true, - "judgeNotes": "Identified image-transform as the affected function and described the recurring HTTP 503 pattern across the morning of 2026-04-28, covering all 8 gateway failures from 07:00Z to 12:00Z." + "name": "supabase db diff used to generate the migration", + "passed": true }, { - "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": false, - "judgeNotes": "Although the assistant correctly attributes the 503s to the gateway/Edge invocation path and grounds this in valid observations, it recommends redeploying the functions as a remediation, which the rubric explicitly lists as a fail condition." + "name": "schema file updated to include description column", + "passed": true }, { - "name": "recommended a concrete next step", - "passed": true, - "judgeNotes": "The assistant recommended concrete next steps including redeploying Edge Functions, checking Supabase incidents/status, adding retry/backoff, and opening a support ticket with exact timestamps and deployment IDs." + "name": "a new migration was generated for the change", + "passed": true + }, + { + "name": "description column exists in the live database", + "passed": true } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"local development database migrations alter table add column db diff\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations", + "title": "Database Migrations" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" + }, + { + "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", + "title": "Declarative database schemas" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", + "title": "Migrate from Postgres to Supabase" + } + ], + "resultChars": 90168 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"set -e\ncurl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|migration|schema' | head -n 80 || true\nprintf '\\\\n--- FILES ---\\\\n'\nfind .. -name AGENTS.md -print\nprintf '\\\\n'\nrg --files -g 'supabase/**' -g '\"'!supabase/.temp/**'\"' | sort\nprintf '\\\\n--- CONFIG ---\\\\n'\nif [ -f supabase/config.toml ]; then rg -n 'schema_paths|major_version|project_id' supabase/config.toml; fi\nprintf '\\\\n--- PRODUCTS REFERENCES ---\\\\n'\nrg -n -i 'create table.*products|public\\\\.products|\\\\bproducts\\\\b' supabase . --glob '\"'!node_modules/**'\"' --glob '\"'!.git/**'\"' | head -n 200\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 13391 + } + ] }, - "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", - "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/investigate-reliability-003-edge-function-5xx-correlation.json" + "prompt": "Add a description text column to the `products` table in my local Supabase stack", + "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6/build-cli-002-declarative-schema.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "resolve-dataapi-001-empty-results", - "stage": "resolve", + "eval": "build-cli-003-pg-cron-queue-workflow", + "stage": "build", "product": [ - "data-api", "database", - "auth" + "edge-functions", + "cron", + "queues" ], "topic": [ - "rls", + "sql", "sdk" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", "passed": true, "checks": [ { - "name": "RLS still enabled on bookmarks", - "passed": true - }, - { - "name": "user A reads own bookmarks", - "passed": true - }, - { - "name": "user B cannot read user A bookmarks", - "passed": true - }, - { - "name": "anon reads no bookmarks", - "passed": true - }, - { - "name": "user A can save a new bookmark", - "passed": true + "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", + "passed": true, + "notes": "schedule='* * * * *', active=true" }, { - "name": "user B cannot insert a bookmark as user A", - "passed": true + "name": "cron command enqueues to the 'tasks' queue", + "passed": true, + "notes": "queue depth 0 -> 1" }, { - "name": "diagnosed RLS and added owner-scoped policies", + "name": "process-tasks function drains the queue", "passed": true, - "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies causing deny-all/empty Data API results, kept RLS enabled, and created authenticated SELECT and INSERT owner-scoped policies using auth.uid() = user_id, including WITH CHECK for INSERT. Extra update/delete owner policies do not violate the rubric." + "notes": "function removed the seeded message (id 4) from the queue" } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- config.toml ---'; sed -n '1,260p' supabase/config.toml; printf '%s\\\\n' '--- migration ---'; cat supabase/migrations/20260101000000_enable_cron_and_queues.sql; printf '%s\\\\n' '--- directories ---'; find supabase -maxdepth 4 -type f -o -type d | sort; printf '%s\\\\n' '--- relevant postgres references ---'; find .agents/skills/supabase-postgres-best-practices/references -maxdepth 1 -type f -printf '%f\\\\n' | sort | rg '\"'^(advanced|schema|security|lock)'\"' | head -80; printf '%s\\\\n' '--- changelog scan ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|queue|pgmq|cron' | head -160\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 14934 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase Queues pgmq create queue send read delete Edge Function cron schedule every minute\", limit: 8) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, + { + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" + }, + { + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" + }, + { + "url": "https://supabase.com/docs/guides/cron/quickstart", + "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/cron/install", + "title": "Install" + }, + { + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "title": "Consuming Supabase Queue Messages with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" + } + ], + "resultChars": 44996 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase Cron schedule SQL cron.schedule every minute jobname\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/cron/quickstart", + "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_cron", + "title": "pg_cron: Schedule Recurring Jobs with Cron Syntax in Postgres" + }, + { + "url": "https://supabase.com/docs/guides/cron/install", + "title": "Install" + } + ], + "resultChars": 26100 + } + ] }, - "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", - "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-dataapi-001-empty-results.json" + "sourcePath": "codex-gpt-5.6/build-cli-003-pg-cron-queue-workflow.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "resolve-database-001-migration-history-mismatch", - "stage": "resolve", + "eval": "build-dataapi-001-relational-report", + "stage": "build", "product": [ + "data-api", "database" ], "topic": [ - "migrations" + "sdk" ], "suite": "benchmark", "interface": "cli", - "passed": true, + "cliVersion": "2.109.1", + "passed": false, "checks": [ { - "name": "the avatar_url column is applied on the hosted profiles table", - "passed": true - }, - { - "name": "migration 20240220000000 is recorded in the remote history", - "passed": true + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" }, - { - "name": "remote migration history matches local migration files", - "passed": true + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" }, { - "name": "local migrations are a valid reconciled sequence", - "passed": true + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" }, { - "name": "production profile data is intact (not reset)", - "passed": true + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" }, { - "name": "the avatar migration and history reconciliation were done via the Supabase CLI", + "name": "report queries via the Data API, not raw SQL", "passed": true, - "judgeNotes": "Avatar migration was applied through Supabase CLI with `supabase db push --db-url 'postgresql://postgres:postgres@172.17.0.1:41925/postgres?sslmode=disable'` (#34), which output `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration history was reconciled by adding the local migration file `supabase/migrations/20240115000000_add_profile_bio.sql` (#31), after which `supabase migration list` showed local and remote matched for `20240115000000` (#32), allowing the push to proceed. Only read-only `psql` inspections were used; no prohibited direct SQL mutation or prepared-statement workaround was seen." + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- files ---'; rg --files -g 'AGENTS.md' -g 'package.json' -g 'app/**' -g 'supabase/**' | sort; printf '%s\\\\n' '--- report ---'; sed -n '1,240p' app/report.mjs; printf '%s\\\\n' '--- package ---'; test -f package.json && cat package.json || true; printf '%s\\\\n' '--- config ---'; sed -n '1,220p' supabase/config.toml; printf '%s\\\\n' '--- migrations/seeds ---'; for f in supabase/migrations/*.sql supabase/seed.sql; do test -f \\\"\"'$f\" && { echo \"### $f\"; sed -n '\"'1,260p' \\\"\"'$f\"; }; done; printf '\"'%s\\\\n' '--- changelog relevant lines ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -m 20 'breaking-change|supabase-js|postgrest|data api' || true\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 10695 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"supabase-js select nested relationships service role secret key createClient Node\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/performing-administration-tasks-on-the-server-side-with-the-servicerole-secret-BYM4Fa", + "title": "Performing administration tasks on the server side with a secret key" + }, + { + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-regenerateclientsecret" + }, + { + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-updateclient" + } + ], + "resultChars": 15624 + } + ] }, - "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", - "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", "attempts": 2, - "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-database-001-migration-history-mismatch.json" + "sourcePath": "codex-gpt-5.6/build-dataapi-001-relational-report.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "resolve-performance-001-slow-query-cpu-spike", - "stage": "resolve", + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", "product": [ + "data-api", "database" ], "topic": [ - "observability", - "sql" + "sdk" ], "suite": "benchmark", - "interface": "mcp", - "passed": true, + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, "checks": [ { - "name": "inspected pg_stat_statements for query performance", - "passed": true + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" }, { - "name": "ran EXPLAIN on the expensive query", - "passed": true + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" }, { - "name": "created index covering user_id and created_at", - "passed": true + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" }, { - "name": "query plan uses an index and avoids sequential scan", - "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_desc_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" }, { - "name": "inserts still work", - "passed": true + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"supabase-js select foreign tables nested relationships filter referenced table column Node createClient service role\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/ai/engineering-for-scale", + "title": "Engineering for Scale" + }, + { + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" + }, + { + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/database/joins-and-nesting", + "title": "Querying Joins and Nested tables" + } + ], + "resultChars": 41919 + } + ] }, - "prompt": "My database CPU keeps spiking and the app gets slow when loading recent events for a user. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", - "promptSourcePath": "evals/resolve-performance-001-slow-query-cpu-spike/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-performance-001-slow-query-cpu-spike.json" + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/build-dataapi-002-restock-alert-report.json" }, { - "experiment": "codex-gpt-5.4-mini-no-skills", - "experimentSuite": "no-skills", + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", - "modelId": "gpt-5.4-mini", + "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "resolve-security-002-rls-cross-tenant-leak", - "stage": "resolve", + "eval": "build-database-001-migrate-postgres-to-supabase", + "stage": "build", "product": [ - "database", - "auth" + "database" ], "topic": [ - "rls", - "security" + "migrations" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", "passed": true, "checks": [ { - "name": "RLS enabled on notes", - "passed": true - }, - { - "name": "tenant A sees only org A notes", - "passed": true - }, - { - "name": "tenant B cannot read org A notes", - "passed": true - }, - { - "name": "tenant A author can update own note", - "passed": true - }, - { - "name": "tenant B cannot update org A note", + "name": "all 3 tables exist (teams, members, tasks)", "passed": true }, { - "name": "tenant B author can delete own note", + "name": "row counts match (teams=5, members=10, tasks=13)", "passed": true }, { - "name": "tenant B cannot delete org A note", + "name": "foreign key constraints survived the restore", "passed": true }, { - "name": "tenant A can insert note in own org", + "name": "tasks_team_status_idx index survived the restore", "passed": true }, { - "name": "tenant B cannot insert into org A", + "name": "sequences synced (next insert won't conflict with existing IDs)", "passed": true } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"restore pg_restore existing database dump local Supabase CLI custom format roles extensions\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on CLICommandReference { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", + "title": "Migrate from Postgres to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" + }, + { + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/transferring-from-cloud-to-self-host-in-supabase-2oWNvW", + "title": "Transferring from platform to self-hosted Supabase" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/custom-postgres-extensions", + "title": "Custom Postgres Extensions" + } + ], + "resultChars": 50650 + } + ] }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", + "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-security-002-rls-cross-tenant-leak.json" + "sourcePath": "codex-gpt-5.6/build-database-001-migrate-postgres-to-supabase.json" }, { "experiment": "codex-gpt-5.6", @@ -12814,60 +13757,46 @@ "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "build-auth-001-email-password-flow", + "eval": "build-functions-004-service-role-bypass", "stage": "build", "product": [ + "edge-functions", "auth", "database" ], "topic": [ - "sdk", - "rls" + "rls", + "security", + "sdk" ], "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", + "interface": "mcp", "passed": true, "checks": [ { - "name": "auth module loads and the driver completes", - "passed": true, - "notes": "driver produced a result" - }, - { - "name": "signUp creates the account and returns its user id", - "passed": true, - "notes": "db user 54178c78-bfc4-4a47-8331-65384278fdef, signUp returned {\"userId\":\"54178c78-bfc4-4a47-8331-65384278fdef\"}" - }, - { - "name": "signup metadata reaches the profile (display name)", - "passed": true, - "notes": "profiles.display_name = \"Alex Doe\"" - }, - { - "name": "wrong password is rejected gracefully (no throw, no session)", + "name": "rejects missing auth", "passed": true, - "notes": "{\"error\":\"Invalid login credentials\"}" + "notes": "status=401" }, { - "name": "signIn with the right password returns the user id", + "name": "user A reads own note", "passed": true, - "notes": "{\"userId\":\"54178c78-bfc4-4a47-8331-65384278fdef\"}" + "notes": "status=200" }, { - "name": "getMyProfile returns the signed-in user's profile", + "name": "reads only with the caller's JWT", "passed": true, - "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + "notes": "bearer_tokens=2, all_match=true" }, { - "name": "app code does not use the secret / service-role key", + "name": "user A cannot force-read user B note", "passed": true, - "notes": "no secret-key references found" + "notes": "status=200" }, { - "name": "implementation uses @supabase/supabase-js", + "name": "user B cannot force-read user A note", "passed": true, - "notes": "imports found in: app/src/auth.mjs" + "notes": "status=200" } ], "skills": { @@ -12882,112 +13811,78 @@ }, "docs": { "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js signUp email password options data user metadata signInWithPassword getUser select single profile\", limit: 5) { nodes { ... on ClientLibraryFunctionReference { title href content language methodName } ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - } - ], - "resultChars": 80822 - }, { "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- changelog auth mentions ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|auth|supabase-js' | sed -n '1,220p'; printf '%s\\\\n' '--- auth.mjs ---'; cat app/src/auth.mjs; printf '%s\\\\n' '--- app directory ---'; find app -maxdepth 3 -type f -o -type l | sort | xargs -r ls -l; printf '%s\\\\n' '--- service status ---'; supabase status -o env\"", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- security rules ---'; rg --files /tmp/sandbox-f5d66cbb/.agents/skills/supabase-postgres-best-practices/references | rg '/security-|/query-' | sort; printf '%s\\\\n' '--- changelog auth/function mentions ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -m 20 'edge function|jwt|auth|breaking-change'\"", "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 1410 + "resultChars": 4608 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript signUp email password options data display_name\", limit: 10) { nodes { ... on ClientLibraryFunctionReference { title href content language methodName } ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"Edge Functions authenticate user Authorization header getUser createClient RLS service role security\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signup" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", - "title": "Passwordless email logins" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/reference/csharp/auth-signinwithpassword", - "title": "SignIn(email, password)" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-verifyotp" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-passkey-list" + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-signinanonymously" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/reference/dart/auth-signup" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" } ], - "resultChars": 19311 + "resultChars": 67563 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript auth getUser current user select maybeSingle profiles table\", limit: 8) { nodes { ... on ClientLibraryFunctionReference { title href content language methodName } ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"Edge Functions default secrets SUPABASE_ANON_KEY SUPABASE_PUBLISHABLE_KEYS 2026\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/auth-getuser" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-getuserbyid" - }, - { - "url": "https://supabase.com/docs/reference/javascript/select" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-listusers" + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-createuser" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-updateuser" + "url": "https://supabase.com/docs/guides/functions/storage-caching", + "title": "Integrating with Supabase Storage" }, { - "url": "https://supabase.com/docs/reference/javascript/using-modifiers-maybesingle" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" } ], - "resultChars": 7456 + "resultChars": 36970 } ] }, - "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", - "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-auth-001-email-password-flow.json" + "sourcePath": "codex-gpt-5.6/build-functions-004-service-role-bypass.json" }, { "experiment": "codex-gpt-5.6", @@ -12998,50 +13893,67 @@ "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "build-cli-001-bootstrap-app", + "eval": "build-functions-005-dual-auth-user-secret", "stage": "build", "product": [ - "database", - "data-api" + "edge-functions", + "auth", + "database" ], "topic": [ - "migrations", - "rls" + "sdk", + "rls", + "security" ], "suite": "benchmark", "interface": "cli", + "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "supabase project initialised (supabase/config.toml exists)", - "passed": true + "name": "seed rows present", + "passed": true, + "notes": "found 2/2 seeded rows" }, { - "name": "todos table is created by a migration file", - "passed": true + "name": "rejects request with no credentials", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { - "name": "todos table exists with at least 2 seeded rows", + "name": "user with JWT reads only their own rows", "passed": true, - "notes": "found 2 rows" + "notes": "status 200: [{\"user_id\":\"b1f73f34-4420-483e-843c-3fb748549436\",\"metric\":\"steps_a_mssypzfw\",\"value\":111}]" }, { - "name": "row level security is enabled on todos", - "passed": true + "name": "user cannot read another user's rows by passing user_id", + "passed": true, + "notes": "status 200: [{\"user_id\":\"b1f73f34-4420-483e-843c-3fb748549436\",\"metric\":\"steps_a_mssypzfw\",\"value\":111}]" }, { - "name": "a SELECT policy targets the authenticated role", - "passed": true + "name": "service key bypasses RLS to read the target user's rows", + "passed": true, + "notes": "status 200: [{\"user_id\":\"ac66cd5d-b572-4aeb-9873-0535838a2c7e\",\"metric\":\"steps_b_mssypzfw\",\"value\":222}]" }, { - "name": "REST API returns no todos to anonymous requests", + "name": "non-service key is not granted service access", "passed": true, - "notes": "0 rows" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { - "name": "REST API returns the todos to authenticated requests", + "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "2 rows" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "a user token in the apikey slot is not treated as the service key", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "implementation uses @supabase/server", + "passed": true, + "notes": "imports @supabase/server / withSupabase" } ], "skills": { @@ -13056,62 +13968,110 @@ }, "docs": { "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Functions JWT verification service role apikey header Authorization serve verify_jwt config\", limit: 8) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" + }, + { + "url": "https://supabase.com/docs/guides/functions/status-codes", + "title": "Status codes" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + } + ], + "resultChars": 62191 + }, { "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- CLAUDE SUPABASE ---'; sed -n '1,240p' .claude/skills/supabase/SKILL.md; printf '%s\\\\n' '--- CLAUDE POSTGRES ---'; sed -n '1,220p' .claude/skills/supabase-postgres-best-practices/SKILL.md; printf '%s\\\\n' '--- SECURITY REFS ---'; find .agents/skills/supabase-postgres-best-practices/references -maxdepth 1 -type f -printf '%f\\\\n' | sort | rg 'security|schema'; printf '%s\\\\n' '--- RLS REFERENCES ---'; cat .agents/skills/supabase-postgres-best-practices/references/security-rls-performance.md 2>/dev/null; cat .agents/skills/supabase-postgres-best-practices/references/security-rls.md 2>/dev/null; printf '%s\\\\n' '--- CLI VERSION ---'; supabase --version; printf '%s\\\\n' '--- GIT ---'; git status --short --branch 2>/dev/null || true; printf '%s\\\\n' '--- CHANGELOG MATCHES ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|row level|rls|local development|cli' | head -n 120\"", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking|edge function|api key|secret key|jwt' | head -80; printf '\\\\n--- config ---\\\\n'; sed -n '1,240p' supabase/config.toml; printf '\\\\n--- migration ---\\\\n'; cat supabase/migrations/0000_stats_schema.sql; printf '\\\\n--- tree ---\\\\n'; find supabase -maxdepth 4 -type f -not -path '*/.temp/*' -print | sort; printf '\\\\n--- security refs ---\\\\n'; cat .agents/skills/supabase-postgres-best-practices/references/security-rls.md 2>/dev/null || true; cat .agents/skills/supabase-postgres-best-practices/references/security-privilege.md 2>/dev/null || true\"", "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 8747 + "resultChars": 18592 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Data API expose table grant anon authenticated RLS select policy authenticated local development CLI migrations db reset\", limit: 8) { nodes { __typename title href content } } }", + "query": "query { searchDocs(query: \"@supabase/server withSupabase auth user or secret multiple auth modes Edge Function\", limit: 10) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", + "title": "Which package to use" }, { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" }, { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" + "url": "https://supabase.com/docs/guides/functions/websockets", + "title": "Handling WebSockets" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/supavisor-faq-YyP5tI", + "title": "Supavisor FAQ" + }, + { + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" }, { - "url": "https://supabase.com/docs/reference/cli/supabase-db-reset", - "title": "Resets the local database to current migrations" + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" } ], - "resultChars": 124182 + "resultChars": 121827 } ] }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", + "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", + "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-cli-001-bootstrap-app.json" + "sourcePath": "codex-gpt-5.6/build-functions-005-dual-auth-user-secret.json" }, { "experiment": "codex-gpt-5.6", @@ -13122,34 +14082,57 @@ "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "build-cli-002-declarative-schema", + "eval": "build-storage-001-private-bucket-access", "stage": "build", "product": [ + "storage", "database" ], "topic": [ - "declarative-schema", - "migrations" + "rls", + "sdk" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "supabase db diff used to generate the migration", + "name": "bucket user-files exists", + "passed": true + }, + { + "name": "bucket user-files is private", + "passed": true + }, + { + "name": "RLS still enabled on storage.objects", + "passed": true + }, + { + "name": "user A lists only own files", + "passed": true, + "notes": "saw: 01a0005d-e448-736c-9743-fe003b0efc12/receipt-alpha.pdf, 01a0005d-e448-736c-9743-fe003b0efc12/receipt-beta.pdf" + }, + { + "name": "user B cannot read user A files", "passed": true }, { - "name": "schema file updated to include description column", + "name": "anon reads no files", "passed": true }, { - "name": "a new migration was generated for the change", + "name": "user A can upload into own folder", "passed": true }, { - "name": "description column exists in the live database", + "name": "user B cannot upload into user A folder", "passed": true + }, + { + "name": "configured private per-user storage access", + "passed": true, + "judgeNotes": "Meets rubric: private user-files bucket, RLS remains enabled, authenticated owner-scoped SELECT and INSERT policies, and createSignedUrl with expiry for temporary sharing." } ], "skills": { @@ -13164,50 +14147,99 @@ }, "docs": { "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- relevant best-practice files ---'; rg --files .claude/skills/supabase-postgres-best-practices/references | rg 'security|rls|index'; printf '%s\\\\n' '--- changelog storage mentions ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|storage' | head -120\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 9620 + }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase declarative database schemas generate migration db diff local\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Storage access control RLS policies foldername auth.uid createSignedUrl JavaScript private bucket\", limit: 8) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", - "title": "Declarative database schemas" + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" + "url": "https://supabase.com/docs/guides/security/product-security", + "title": "Secure configuration of Supabase products" }, { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" }, { - "url": "https://supabase.com/docs/guides/deployment/managing-environments", - "title": "Managing Environments" + "url": "https://supabase.com/docs/guides/auth/third-party/clerk", + "title": "Clerk" + }, + { + "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", + "title": "Custom Roles" + }, + { + "url": "https://supabase.com/docs/guides/realtime/getting_started", + "title": "Getting Started with Realtime" } ], - "resultChars": 82866 + "resultChars": 49465 }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|declarative|migration|db diff' | head -80; printf '\\\\n--- CONFIG ---\\\\n'; sed -n '1,220p' supabase/config.toml; printf '\\\\n--- SCHEMA ---\\\\n'; cat supabase/schemas/products.sql; printf '\\\\n--- MIGRATION ---\\\\n'; cat supabase/migrations/20260101000000_create_products.sql; printf '\\\\n--- STATUS ---\\\\n'; supabase status || true\"", + "source": "search_docs", + "query": "query { searchDocs(query: \"storage.foldername auth.uid first folder policy example createSignedUrl expiresIn JavaScript\", limit: 10) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" + }, + { + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" + }, + { + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" + }, + { + "url": "https://supabase.com/docs/reference/javascript/file-buckets-uploadtosignedurl" + }, + { + "url": "https://supabase.com/docs/guides/storage/serving/downloads", + "title": "Serving assets from Storage" + }, + { + "url": "https://supabase.com/docs/guides/storage/uploads/resumable-uploads", + "title": "Resumable Uploads" + }, + { + "url": "https://supabase.com/docs/reference/dart/file-buckets-createsignedurl" + }, + { + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" + }, + { + "url": "https://supabase.com/docs/reference/dart/file-buckets-createsigneduploadurl" + }, + { + "url": "https://supabase.com/docs/reference/javascript/file-buckets-createbucket" } ], - "resultChars": 8129 + "resultChars": 27849 } ] }, - "prompt": "Add a description text column to the `products` table in my local Supabase stack", - "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", + "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", + "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-cli-002-declarative-schema.json" + "sourcePath": "codex-gpt-5.6/build-storage-001-private-bucket-access.json" }, { "experiment": "codex-gpt-5.6", @@ -13218,36 +14250,33 @@ "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "build-cli-003-pg-cron-queue-workflow", + "eval": "build-tests-001-rls-tenant-isolation", "stage": "build", "product": [ - "database", - "edge-functions", - "cron", - "queues" + "database" ], "topic": [ - "sql", - "sdk" + "tests", + "rls" ], "suite": "benchmark", "interface": "cli", "passed": true, "checks": [ { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", + "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "schedule='* * * * *', active=true" + "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" }, { - "name": "cron command enqueues to the 'tasks' queue", + "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "queue depth 0 -> 1" + "notes": "9 passed, 0 failed" }, { - "name": "process-tasks function drains the queue", + "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "notes": "function removed the seeded message (id 4) from the queue" + "judgeNotes": "The agent correctly identified `posts` as having the tenant isolation flaw, explicitly stated that `notes` was correctly isolated, and grounded the conclusion in runtime/pgTAP-style verification results." } ], "skills": { @@ -13264,120 +14293,48 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Queues pgmq create queue send read delete Edge Function cron schedule every minute\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" - }, - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - } - ], - "resultChars": 44996 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Cron schedule database function cron.schedule pgmq send queue\", limit: 6) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"row level security testing policies auth.uid tenant isolation pgTAP\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", - "title": "pg_cron debugging guide" + "url": "https://supabase.com/docs/guides/local-development/testing/overview", + "title": "Testing Overview" }, { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" + "url": "https://supabase.com/docs/guides/database/extensions/pgtap", + "title": "pgTAP: Unit Testing" }, { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" + "url": "https://supabase.com/docs/guides/database/testing", + "title": "Testing Your Database" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" } ], - "resultChars": 35015 + "resultChars": 56902 }, { "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- CHANGELOG MATCHES ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|queue|cron|edge function' | head -n 160 || true; printf '%s\\\\n' '--- CONFIG RELEVANT ---'; rg -n '\"'^'\"\\\\[|enabled|verify_jwt|additional_search_path|port' supabase/config.toml; printf '%s\\\\n' '--- MIGRATION HEAD ---'; sed -n '1,100p' supabase/migrations/20260101000000_enable_cron_and_queues.sql; printf '%s\\\\n' '--- MIGRATION TAIL ---'; tail -n 80 supabase/migrations/20260101000000_enable_cron_and_queues.sql; printf '%s\\\\n' '--- MIGRATION SYMBOLS ---'; rg -n 'create extension|pgmq.create|cron.schedule|pgmq_public|create schema' supabase/migrations/20260101000000_enable_cron_and_queues.sql\"", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- CHANGELOG TAG SCAN ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -n 'breaking-change|row level|rls|pgtap|database test' | head -80 || true; printf '%s\\\\n' '--- SECURITY REFERENCES ---'; for f in .agents/skills/supabase-postgres-best-practices/references/security-*.md; do echo \\\"### \"'$f\"; sed -n '\"'1,220p' \\\"\"'$f\"; done; printf '\"'%s\\\\n' '--- CONFIG ---'; sed -n '1,240p' supabase/config.toml; printf '%s\\\\n' '--- MIGRATION ---'; sed -n '1,320p' supabase/migrations/20240101000000_init.sql; printf '%s\\\\n' '--- TREE ---'; find . -maxdepth 4 -type f -not -path './.git/*' | sort\"", "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 19533 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions SUPABASE_URL SUPABASE_SERVICE_ROLE_KEY environment variables local serve createClient\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/unit-test", - "title": "Testing your Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - } - ], - "resultChars": 40557 + "resultChars": 27513 } ] }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", + "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-cli-003-pg-cron-queue-workflow.json" + "sourcePath": "codex-gpt-5.6/build-tests-001-rls-tenant-isolation.json" }, { "experiment": "codex-gpt-5.6", @@ -13388,44 +14345,50 @@ "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "build-dataapi-001-relational-report", + "eval": "build-vectors-001-rag-with-permissions", "stage": "build", "product": [ - "data-api", - "database" + "database", + "vectors" ], "topic": [ - "sdk" + "sql", + "rls" ], "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", + "interface": "mcp", "passed": true, "checks": [ { - "name": "report runs and prints JSON", + "name": "document_sections.embedding is vector(384)", "passed": true, - "notes": "exit 0" + "notes": "vector(384)" }, { - "name": "report numbers match the database (per customer, sorted)", + "name": "HNSW index on the embedding column", "passed": true, - "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" }, { - "name": "tables stay locked down (publishable key reads nothing)", + "name": "index operator class matches the search operator", "passed": true, - "notes": "publishable read errored: permission denied for table customers" + "notes": "function operators: <#>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" }, { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/report.mjs" + "name": "user A search returns only own sections, best match first", + "passed": true }, { - "name": "report queries via the Data API, not raw SQL", - "passed": true, - "notes": "no psql / raw Postgres driver usage found" + "name": "user B search returns only own sections, best match first", + "passed": true + }, + { + "name": "user A reads only own sections through the API", + "passed": true + }, + { + "name": "user A reads only own documents through the API", + "passed": true } ], "skills": { @@ -13442,46 +14405,96 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase javascript select nested foreign tables service role secret key createClient Node\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"pgvector semantic search gte-small vector dimensions HNSW match_documents RPC RLS\", limit: 8) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" + }, + { + "url": "https://supabase.com/docs/guides/ai/hybrid-search", + "title": "Hybrid search" + }, + { + "url": "https://supabase.com/docs/guides/ai/vector-indexes", + "title": "Vector indexes" }, { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/ai/going-to-prod", + "title": "Going to Production" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/performing-administration-tasks-on-the-server-side-with-the-servicerole-secret-BYM4Fa", - "title": "Performing administration tasks on the server side with a secret key" + "url": "https://supabase.com/docs/guides/ai/vector-indexes/ivf-indexes", + "title": "IVFFlat indexes" }, { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-updateclient" + "url": "https://supabase.com/docs/guides/ai/vector-indexes/hnsw-indexes", + "title": "HNSW indexes" + }, + { + "url": "https://supabase.com/docs/guides/ai/vector-columns", + "title": "Vector columns" } ], - "resultChars": 22127 + "resultChars": 73600 }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- changelog relevant scan ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -n -m 20 'breaking-change|supabase-js|secret key|postgrest|data api' || true; printf '%s\\\\n' '--- root listing ---'; ls -la; printf '%s\\\\n' '--- app listing ---'; find app -maxdepth 2 -type f -print -exec sed -n '1,80p' {} \\\\;; printf '%s\\\\n' '--- node/npm ---'; node --version; npm --version; npm root -g; printf '%s\\\\n' '--- installed candidates ---'; find . -maxdepth 3 -type d \\\\( -name '@supabase' -o -name 'node_modules' \\\\) -print; printf '%s\\\\n' '--- supabase status env ---'; supabase status -o env\"", + "source": "web_search", + "query": "https://supabase.com/changelog.md", "pages": [ { "url": "https://supabase.com/changelog.md" } + ] + }, + { + "source": "web_search", + "query": "site:supabase.com/changelog pgvector semantic search breaking change Supabase", + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"row level security SQL function security invoker RPC auth.uid vector index hnsw inner product\", limit: 6) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/ai/vector-indexes/hnsw-indexes", + "title": "HNSW indexes" + }, + { + "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0003_auth_rls_initplan", + "title": "Database Advisor: Lint 0003_auth_rls_initplan" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" + }, + { + "url": "https://supabase.com/docs/guides/database/tables", + "title": "Tables and Data" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/wrappers/overview", + "title": "Foreign Data Wrappers" + } ], - "resultChars": 1554 + "resultChars": 88944 } ] }, - "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", - "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.6/build-dataapi-001-relational-report.json" + "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", + "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6/build-vectors-001-rag-with-permissions.json" }, { "experiment": "codex-gpt-5.6", @@ -13492,44 +14505,30 @@ "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "build-dataapi-002-restock-alert-report", - "stage": "build", + "eval": "deploy-database-001-prometheus-metrics", + "stage": "deploy", "product": [ - "data-api", "database" ], "topic": [ - "sdk" + "observability" ], "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { - "name": "report runs and prints JSON", - "passed": true, - "notes": "exit 0" - }, - { - "name": "alerts match the database (below threshold, sorted)", - "passed": true, - "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + "name": "preserved existing app scrape job", + "passed": true }, { - "name": "tables stay locked down (publishable key reads nothing)", + "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "notes": "publishable read errored: permission denied for table inventory" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + "judgeNotes": "Meets rubric: preserves app scrape, adds Supabase HTTPS scrape at /customer/v1/privileged/metrics with Basic Auth password_file, targets .supabase.co, and wires the password file via a Compose secret." }, { - "name": "report queries via the Data API, not raw SQL", + "name": "documented live deployment and verification steps", "passed": true, - "notes": "no psql / raw Postgres driver usage found" + "judgeNotes": "README includes Secret API key creation, local Docker secret file placement, Compose start/recreate steps, and concrete verification via Prometheus targets. Endpoint/auth and secret setup match the provided configuration." } ], "skills": { @@ -13538,56 +14537,341 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js select nested foreign tables createClient secret key Node backend order results\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase project metrics Prometheus endpoint authentication service role monitoring observability\", limit: 8) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } ... on TroubleshootingGuide { title href content } ... on ManagementApiReference { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/ai/engineering-for-scale", - "title": "Engineering for Scale" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", + "title": "Metrics API" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics#additional-resources", + "title": "Additional resources" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics#choose-your-monitoring-stack", + "title": "Choose your monitoring stack" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics#what-you-can-do-with-the-metrics-api", + "title": "What you can do with the Metrics API" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic#2-secure-the-credentials", + "title": "2. Secure the credentials" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic#components", + "title": "Components" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic#1-define-the-scrape-job", + "title": "1. Define the scrape job" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic#collector-specific-notes", + "title": "Collector-specific notes" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic#5-multi-project-setups", + "title": "5. Multi-project setups" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic#4-alerts-and-automation", + "title": "4. Alerts and automation" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic#3-downstream-dashboards", + "title": "3. Downstream dashboards" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#manual-setup", + "title": "Manual setup" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#installation", + "title": "Installation" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#prerequisites", + "title": "Prerequisites" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#1-create-a-grafana-cloud-stack", + "title": "1. Create a Grafana Cloud stack" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#2-install-the-supabase-integration-for-grafana-cloud", + "title": "2. Install the Supabase integration for Grafana Cloud" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#3-configure-the-supabase-integration", + "title": "3. Configure the Supabase integration" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#4-import-the-supabase-dashboard", + "title": "4. Import the Supabase dashboard" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#5-configure-alerts-optional", + "title": "5. Configure alerts (optional)" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#6-troubleshooting", + "title": "6. Troubleshooting" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted#3-import-supabase-dashboards", + "title": "3. Import Supabase dashboards" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted#4-configure-alerting", + "title": "4. Configure alerting" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted#5-operating-tips", + "title": "5. Operating tips" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted#architecture", + "title": "Architecture" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted#1-deploy-prometheus", + "title": "1. Deploy Prometheus" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted#2-deploy-grafana", + "title": "2. Deploy Grafana" + }, + { + "url": "https://supabase.com/docs/guides/platform/read-replicas", + "title": "Read Replicas" + }, + { + "url": "https://supabase.com/docs/guides/platform/read-replicas#logging", + "title": "Logging" + }, + { + "url": "https://supabase.com/docs/guides/platform/read-replicas#metrics", + "title": "Metrics" + }, + { + "url": "https://supabase.com/docs/guides/platform/read-replicas#centralized-configuration-management", + "title": "Centralized configuration management" + }, + { + "url": "https://supabase.com/docs/guides/platform/read-replicas#pricing", + "title": "Pricing" + }, + { + "url": "https://supabase.com/docs/guides/platform/read-replicas#about-read-replicas", + "title": "About Read Replicas" + }, + { + "url": "https://supabase.com/docs/guides/platform/read-replicas#features", + "title": "Features" + }, + { + "url": "https://supabase.com/docs/guides/platform/read-replicas#dedicated-endpoints", + "title": "Dedicated endpoints" + }, + { + "url": "https://supabase.com/docs/guides/platform/read-replicas#dedicated-connection-pool", + "title": "Dedicated connection pool" + }, + { + "url": "https://supabase.com/docs/guides/platform/read-replicas#api-load-balancer", + "title": "API load balancer" + }, + { + "url": "https://supabase.com/docs/guides/platform/read-replicas#querying-through-the-sql-editor", + "title": "Querying through the SQL editor" + }, + { + "url": "https://supabase.com/docs/guides/database/connection-management", + "title": "Connection management" + }, + { + "url": "https://supabase.com/docs/guides/database/connection-management#observing-live-connections", + "title": "Observing live connections" + }, + { + "url": "https://supabase.com/docs/guides/database/connection-management#grafana-dashboard", + "title": "Grafana Dashboard" + }, + { + "url": "https://supabase.com/docs/guides/database/connection-management#dashboard-monitoring-charts", + "title": "Dashboard monitoring charts" + }, + { + "url": "https://supabase.com/docs/guides/database/connection-management#capturing-historical-usage", + "title": "Capturing historical usage" + }, + { + "url": "https://supabase.com/docs/guides/database/connection-management#monitoring-connections", + "title": "Monitoring connections" + }, + { + "url": "https://supabase.com/docs/guides/database/connection-management#connections", + "title": "Connections" + }, + { + "url": "https://supabase.com/docs/guides/database/connection-management#configuring-supavisors-pool-size", + "title": "Configuring Supavisor's pool size" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/reports", + "title": "Reports" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/reports#memory-commitment", + "title": "Memory commitment" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/reports#memory-usage", + "title": "Memory usage" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/reports#advanced-telemetry", + "title": "Advanced Telemetry" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/reports#database", + "title": "Database" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/reports#auth-api-gateway", + "title": "Auth API Gateway" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/reports#auth", + "title": "Auth" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/reports#api-gateway", + "title": "API gateway" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/reports#using-reports", + "title": "Using reports" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/reports#postgrest", + "title": "PostgREST" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/reports#realtime", + "title": "Realtime" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/reports#realtime-api-gateway", + "title": "Realtime API Gateway" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/reports#storage", + "title": "Storage" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/reports#edge-functions", + "title": "Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/reports#database-size", + "title": "Database size" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/reports#disk-usage", + "title": "Disk Usage" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/reports#shared-pooler-supavisor-client-connections", + "title": "Shared Pooler (Supavisor) Client Connections" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/reports#dedicated-pooler-pgbouncer-client-connections", + "title": "Dedicated Pooler (PgBouncer) Client Connections" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/reports#database-connections", + "title": "Database connections" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/reports#query-performance", + "title": "Query Performance" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/reports#disk-size", + "title": "Disk size" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/reports#disk-throughput", + "title": "Disk throughput" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/reports#disk-inputoutput-operations-per-second-iops", + "title": "Disk input/output operations per second (IOPS)" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/reports#cpu-usage", + "title": "CPU usage" + }, + { + "url": "https://supabase.com/docs/guides/auth/audit-logs", + "title": "Auth Audit Logs" }, { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" + "url": "https://supabase.com/docs/guides/auth/audit-logs#limitations", + "title": "Limitations" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/performing-administration-tasks-on-the-server-side-with-the-servicerole-secret-BYM4Fa", - "title": "Performing administration tasks on the server side with a secret key" + "url": "https://supabase.com/docs/guides/auth/audit-logs#log-actions-reference", + "title": "Log actions reference" }, { - "url": "https://supabase.com/docs/guides/api/creating-routes", - "title": "Creating API Routes" + "url": "https://supabase.com/docs/guides/auth/audit-logs#log-format", + "title": "Log format" }, { - "url": "https://supabase.com/docs/guides/api/quickstart", - "title": "Build an API route in less than 2 minutes." - } - ], - "resultChars": 27813 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- app/package.json ---'; cat app/package.json; printf '%s\\\\n' '--- git status ---'; git status --short; printf '%s\\\\n' '--- supabase status ---'; supabase status -o env; printf '%s\\\\n' '--- relevant changelog scan ---'; curl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|secret key|supabase-js|postgrest|data api' | head -80\"", - "pages": [ + "url": "https://supabase.com/docs/guides/auth/audit-logs#what-gets-logged", + "title": "What gets logged" + }, { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/auth/audit-logs#configuring-audit-log-storage", + "title": "Configuring audit log storage" + }, + { + "url": "https://supabase.com/docs/guides/auth/audit-logs#storage-options", + "title": "Storage options" } ], - "resultChars": 15363 + "resultChars": 201413 } ] }, - "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", - "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.6/build-dataapi-002-restock-alert-report.json" + "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", + "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6/deploy-database-001-prometheus-metrics.json" }, { "experiment": "codex-gpt-5.6", @@ -13598,36 +14882,34 @@ "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "build-database-001-migrate-postgres-to-supabase", - "stage": "build", + "eval": "deploy-functions-001-edge-function-secrets", + "stage": "deploy", "product": [ - "database" + "edge-functions" ], "topic": [ - "migrations" + "security" ], "suite": "benchmark", "interface": "cli", "passed": true, "checks": [ { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true - }, - { - "name": "row counts match (teams=5, members=10, tasks=13)", + "name": "WEATHER_API_KEY is set as a Function secret on the project", "passed": true }, { - "name": "foreign key constraints survived the restore", - "passed": true + "name": "the weather function is deployed to the project", + "passed": true, + "notes": "status ACTIVE" }, { - "name": "tasks_team_status_idx index survived the restore", - "passed": true + "name": "the weather function reads WEATHER_API_KEY from the environment", + "passed": true, + "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." }, { - "name": "sequences synced (next insert won't conflict with existing IDs)", + "name": "WEATHER_API_KEY value is not committed to the repo", "passed": true } ], @@ -13645,190 +14927,156 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"restore pg_restore local Supabase binary dump roles ownership extensions\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Edge Functions deploy secrets environment variables Deno.env CORS invoke verify_jwt\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", - "title": "Restore a Platform Project to Self-Hosted" + "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", + "title": "Inspecting edge function environment variables" }, { - "url": "https://supabase.com/docs/guides/self-hosting/custom-postgres-extensions", - "title": "Custom Postgres Extensions" + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/transferring-from-cloud-to-self-host-in-supabase-2oWNvW", - "title": "Transferring from platform to self-hosted Supabase" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", - "title": "Migrate from Postgres to Supabase" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/slack-bot-mention", + "title": "Slack Bot Mention Edge Function" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/unable-to-call-edge-function", + "title": "Unable to call Edge Function" } ], - "resultChars": 50510 - } - ] - }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-database-001-migrate-postgres-to-supabase.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-functions-004-service-role-bypass", - "stage": "build", - "product": [ - "edge-functions", - "auth", - "database" - ], - "topic": [ - "rls", - "security", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "rejects missing auth", - "passed": true, - "notes": "status=401" - }, - { - "name": "user A reads own note", - "passed": true, - "notes": "status=200" - }, - { - "name": "reads only with the caller's JWT", - "passed": true, - "notes": "bearer_tokens=2, all_match=true" - }, - { - "name": "user A cannot force-read user B note", - "passed": true, - "notes": "status=200" - }, - { - "name": "user B cannot force-read user A note", - "passed": true, - "notes": "status=200" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ + "resultChars": 57012 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"set -o pipefail\nprintf '%s\\\\n' '--- changelog breaking changes ---'\ncurl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|edge function|secret' | head -80 || true\nprintf '%s\\\\n' '--- repository files ---'\nrg --files -g '\"'!*node_modules*'\"' -g '\"'!*.lock'\"' | sed -n '1,240p'\nprintf '%s\\\\n' '--- git status/remotes ---'\ngit status --short && git remote -v || true\nprintf '%s\\\\n' '--- supabase status/config ---'\nsupabase --version\n[ -f supabase/config.toml ] && sed -n '1,240p' supabase/config.toml || true\nprintf '%s\\\\n' '--- env variable names only ---'\nif [ -f .env ]; then sed -E 's/=.*/=/' .env; else echo 'no root .env'; fi\nprintf '%s\\\\n' '--- weather references ---'\nrg -n -i 'weatherapi|weather|WEATHER_API_KEY' . -g '\"'!*node_modules*'\"' -g '\"'!.git/*'\"' -g '\"'!.env'\"' | head -240 || true\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 699 + }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions authenticate user Authorization header createClient service role getUser RLS\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "query": "query { searchDocs(query: \"Supabase Edge Functions quickstart create function index.ts deploy no verify jwt config.toml per function public browser\", limit: 10) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" + "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", + "title": "Getting Started with Edge Functions (Dashboard)" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/functions/quickstart", + "title": "Getting Started with Edge Functions" }, { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" }, { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" + "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", + "title": "Supabase for Platforms" + }, + { + "url": "https://supabase.com/docs/guides/functions/architecture", + "title": "Edge Functions Architecture" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-functions", + "title": "Manage Supabase Edge functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/development-tips", + "title": "Development tips" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/slack-bot-mention", + "title": "Slack Bot Mention Edge Function" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", + "title": "Resumable WebSockets with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" } ], - "resultChars": 73053 + "resultChars": 78292 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions auth getUser Authorization header SUPABASE_ANON_KEY createClient\", limit: 10) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"Management API invoke deployed Edge Function endpoint test function\", limit: 10) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", + "title": "Getting Started with Edge Functions (Dashboard)" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/guides/functions/architecture", + "title": "Edge Functions Architecture" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" }, { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" + "url": "https://supabase.com/docs/reference/api/v1-deploy-a-function", + "title": "Deploy a function" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/functions/quickstart", + "title": "Getting Started with Edge Functions" }, { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" + "url": "https://supabase.com/docs/guides/functions/deploy", + "title": "Deploy to Production" }, { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" + "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", + "title": "Supabase for Platforms" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" + "url": "https://supabase.com/docs/reference/api/v1-create-a-function", + "title": "Create a function" }, { - "url": "https://supabase.com/docs/guides/functions/storage-caching", - "title": "Integrating with Supabase Storage" - } - ], - "resultChars": 96751 - }, - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ] - }, - { - "source": "web_search", - "query": "site:supabase.com/changelog.md Supabase changelog breaking change Edge Functions auth API keys", - "pages": [] + "url": "https://supabase.com/docs/guides/functions/unit-test", + "title": "Testing your Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/ai-tools/mcp", + "title": "Supabase MCP Server" + } + ], + "resultChars": 106375 } ] }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", + "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", + "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-functions-004-service-role-bypass.json" + "sourcePath": "codex-gpt-5.6/deploy-functions-001-edge-function-secrets.json" }, { "experiment": "codex-gpt-5.6", @@ -13839,67 +15087,35 @@ "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "build-functions-005-dual-auth-user-secret", - "stage": "build", + "eval": "deploy-self-hosting-001-docker-compose", + "stage": "deploy", "product": [ - "edge-functions", + "database", "auth", - "database" + "storage" ], "topic": [ - "sdk", - "rls", - "security" + "self-hosting" ], "suite": "benchmark", "interface": "cli", - "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"c2a94e60-d0fd-41fa-9393-3d88c36ccd1e\",\"metric\":\"steps_a_msj16f6m\",\"value\":111}]" - }, - { - "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: [{\"user_id\":\"c2a94e60-d0fd-41fa-9393-3d88c36ccd1e\",\"metric\":\"steps_a_msj16f6m\",\"value\":111}]" - }, - { - "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"29d6a26d-c31f-496e-a486-a4bd4a0adeee\",\"metric\":\"steps_b_msj16f6m\",\"value\":222}]" - }, - { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", + "passed": true }, { - "name": "rejects an unverified (forged) user token", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", + "passed": true }, { - "name": "a user token in the apikey slot is not treated as the service key", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "name": "secrets rotated off the shipped defaults", + "passed": true }, { - "name": "implementation uses @supabase/server", - "passed": true, - "notes": "imports @supabase/server / withSupabase" + "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", + "passed": true } ], "skills": { @@ -13907,851 +15123,568 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "loaded": [] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions JWT verification authorization header service role apikey header Deno serve createClient auth getUser\", limit: 8) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"self-hosting Docker compose secrets production VPS Supabase docker\", limit: 8) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } ... on CLICommandReference { title href content } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/self-hosting/docker#changing-database-password", + "title": "Changing database password" }, { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" + "url": "https://supabase.com/docs/guides/self-hosting/docker#starting-and-stopping", + "title": "Starting and stopping" }, { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" + "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-supabase-studio-dashboard", + "title": "Accessing Supabase Studio (Dashboard)" }, { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" + "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-postgres", + "title": "Accessing Postgres" }, { - "url": "https://supabase.com/docs/guides/functions/http-methods", - "title": "Routing" + "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-edge-functions", + "title": "Accessing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-apis", + "title": "Accessing APIs" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" - } - ], - "resultChars": 60939 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- CHANGELOG BREAKING TAGS ---'; curl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|edge function|secret key|service.role' | head -80; printf '%s\\\\n' '--- PROJECT FILES ---'; find .. -name AGENTS.md -print; rg --files -g '\"'!node_modules'\"' -g '\"'!vendor'\"' | sort | sed -n '1,240p'; printf '%s\\\\n' '--- SUPABASE CONFIG ---'; test -f supabase/config.toml && sed -n '1,240p' supabase/config.toml || true; printf '%s\\\\n' '--- MIGRATION MATCHES ---'; rg -n \\\"user_stats|create policy|enable row level security\\\" supabase . --glob '*.sql' --glob '\"'!node_modules'\"' || true\"", - "pages": [ + "url": "https://supabase.com/docs/guides/self-hosting/docker#enabling-analytics", + "title": "Enabling analytics" + }, { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 24755 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Securing Edge Functions verify_jwt false custom JWT validation apikey service role key Authorization header\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-https", + "title": "Configuring HTTPS" + }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/self-hosting/docker#managing-the-stack", + "title": "Managing the stack" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#updating", + "title": "Updating" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#uninstalling", + "title": "Uninstalling" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#advanced-topics", + "title": "Advanced topics" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#architecture", + "title": "Architecture" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#setting-database-password", + "title": "Setting database password" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-secrets", + "title": "Configuring secrets" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-supabase-services", + "title": "Configuring Supabase services" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-social-login-oauth-providers", + "title": "Configuring social login (OAuth) providers" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-phone-login-sms-and-mfa", + "title": "Configuring phone login, SMS, and MFA" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-an-email-server", + "title": "Configuring an email server" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-s3-storage", + "title": "Configuring S3 Storage" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#using-file-backend-in-storage-on-macos", + "title": "Using file backend in Storage on macOS" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-supabase-ai-assistant", + "title": "Configuring Supabase AI Assistant" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-postgres-through-supavisor", + "title": "Accessing Postgres through Supavisor" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#exposing-your-postgres-database", + "title": "Exposing your Postgres database" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#setting-log_min_messages-in-postgres", + "title": "Setting log_min_messages in Postgres" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#managing-your-secrets", + "title": "Managing your secrets" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#demo", + "title": "Demo" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#contents", + "title": "Contents" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#before-you-begin", + "title": "Before you begin" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#system-requirements", + "title": "System requirements" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#installing-supabase", + "title": "Installing Supabase" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#quick-start-linux", + "title": "Quick start (Linux)" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#manual-installation", + "title": "Manual installation" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-and-securing-supabase", + "title": "Configuring and securing Supabase" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#generate-keys-and-secrets", + "title": "Generate keys and secrets" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#configure-supabase-urls", + "title": "Configure Supabase URLs" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#where-to-find-your-credentials", + "title": "Where to find your credentials" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#studio-authentication", + "title": "Studio authentication" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#adding-the-new-keys", + "title": "Adding the new keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#new-api-keys-format", + "title": "New API keys format" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#verifying-the-setup", + "title": "Verifying the setup" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#environment-variables-configuration", + "title": "Environment variables configuration" }, { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#differences-from-the-supabase-platform", + "title": "Differences from the Supabase platform" }, { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#backward-compatibility", + "title": "Backward compatibility" }, { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#rotating-the-new-api-keys", + "title": "Rotating the new API keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#regenerating-asymmetric-key-pair", + "title": "Regenerating asymmetric key pair" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#how-it-works", + "title": "How it works" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#what-client-sdk-sends", + "title": "What client SDK sends" }, { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#kong-api-gateway-routing", + "title": "Kong API gateway routing" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#request-flows", + "title": "Request flows" }, { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - } - ], - "resultChars": 121781 - } - ] - }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.6/build-functions-005-dual-auth-user-secret.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-storage-001-private-bucket-access", - "stage": "build", - "product": [ - "storage", - "database" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", - "passed": true - }, - { - "name": "RLS still enabled on storage.objects", - "passed": true - }, - { - "name": "user A lists only own files", - "passed": true, - "notes": "saw: 019fdc8e-c286-72ec-86c2-693b611d4fc9/receipt-alpha.pdf, 019fdc8e-c286-72ec-86c2-693b611d4fc9/receipt-beta.pdf" - }, - { - "name": "user B cannot read user A files", - "passed": true - }, - { - "name": "anon reads no files", - "passed": true - }, - { - "name": "user A can upload into own folder", - "passed": true - }, - { - "name": "user B cannot upload into user A folder", - "passed": true - }, - { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "The answer creates a private user-files bucket, keeps storage.objects RLS enabled, adds authenticated SELECT and INSERT policies scoped to bucket and user-owned path via auth.uid(), and provides supabase-js createSignedUrl with a 15-minute expiry. No fail conditions present." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Storage access control RLS policies foldername auth.uid createSignedUrl private bucket upload download\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#unauthenticated-requests-api-key-only-no-user-session-jwt", + "title": "Unauthenticated requests (API key only, no user session JWT)" + }, { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#authenticated-requests-user-session-jwt", + "title": "Authenticated requests (user session JWT)" }, { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#additional-resources", + "title": "Additional resources" }, { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#before-you-begin", + "title": "Before you begin" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/guides/self-hosting", + "title": "Self-Hosting" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started", - "title": "Getting Started with Realtime" + "url": "https://supabase.com/docs/guides/self-hosting#community-driven-projects", + "title": "Community-driven projects" }, { - "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", - "title": "Custom Roles" + "url": "https://supabase.com/docs/guides/self-hosting#enterprise-self-hosting", + "title": "Enterprise self-hosting" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/why-cant-i-uploadlistetc-my-public-bucket-Z6CmGt", - "title": "Why can't I upload/list/etc my public bucket?" + "url": "https://supabase.com/docs/guides/self-hosting#support-and-community", + "title": "Support and community" }, { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" - } - ], - "resultChars": 46298 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|storage|signed url|rls' | head -80\"", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/self-hosting#telemetry", + "title": "Telemetry" + }, { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 14020 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript createSignedUrl storage expiresIn private bucket signed URL\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/self-hosting#your-responsibilities-when-self-hosting", + "title": "Your responsibilities when self-hosting" + }, { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", - "title": "Storage Image Transformations" + "url": "https://supabase.com/docs/guides/self-hosting#how-self-hosted-supabase-differs", + "title": "How self-hosted Supabase differs" }, { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" + "url": "https://supabase.com/docs/guides/self-hosting#get-started", + "title": "Get started" }, { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurl" + "url": "https://supabase.com/docs/guides/self-hosting#about-self-hosting", + "title": "About self-hosting" }, { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsignedurls" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" }, { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" - } - ], - "resultChars": 25918 - } - ] - }, - "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", - "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-storage-001-private-bucket-access.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-tests-001-rls-tenant-isolation", - "stage": "build", - "product": [ - "database" - ], - "topic": [ - "tests", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "pgTAP test file(s) written under supabase/tests/", - "passed": true, - "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" - }, - { - "name": "pgTAP isolation tests ran and pass", - "passed": true, - "notes": "36 passed, 0 failed" - }, - { - "name": "agent correctly identifies the posts isolation bug from test results", - "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as having a tenant isolation flaw: the policy omits `org_id`, allowing tenant A to see posts from tenant B. It grounds this in reproduced query results and adds pgTAP coverage. It does not blame `notes` instead or dismiss test results." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"row level security testing policies auth.uid tenant isolation pgTAP\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#auth-considerations", + "title": "Auth considerations" + }, { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#postgres-version-compatibility", + "title": "Postgres version compatibility" }, { - "url": "https://supabase.com/docs/guides/local-development/testing/overview", - "title": "Testing Overview" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#troubleshooting", + "title": "Troubleshooting" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pgtap", - "title": "pgTAP: Unit Testing" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#version-mismatches-between-platform-and-self-hosted", + "title": "Version mismatches between platform and self-hosted" }, { - "url": "https://supabase.com/docs/guides/database/testing", - "title": "Testing Your Database" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#extension-not-available", + "title": "Extension not available" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - } - ], - "resultChars": 56902 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- skill relevant sections ---'; rg -n \\\"RLS|policy|test|docs|changelog\\\" .claude/skills/supabase/SKILL.md; printf '%s\\\\n' '--- security references ---'; find .claude/skills/supabase-postgres-best-practices/references -maxdepth 1 -type f | sort | rg 'security|index'; printf '%s\\\\n' '--- read security rules ---'; cat .claude/skills/supabase-postgres-best-practices/references/security-rls.md 2>/dev/null || true; cat .claude/skills/supabase-postgres-best-practices/references/security-rls-performance.md 2>/dev/null || true; printf '%s\\\\n' '--- changelog relevant ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|row.level|rls|pgtap' | head -n 120; printf '%s\\\\n' '--- workspace ---'; find . -name AGENTS.md -print; ls -la; printf '%s\\\\n' '--- repo files ---'; rg --files -g '\"'!node_modules'\"' -g '\"'!vendor'\"' | sed -n '1,240p'\"", - "pages": [ + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#additional-resources", + "title": "Additional resources" + }, { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 17714 - } - ] - }, - "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", - "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-tests-001-rls-tenant-isolation.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-vectors-001-rag-with-permissions", - "stage": "build", - "product": [ - "database", - "vectors" - ], - "topic": [ - "sql", - "rls" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "document_sections.embedding is vector(384)", - "passed": true, - "notes": "vector(384)" - }, - { - "name": "HNSW index on the embedding column", - "passed": true, - "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" - }, - { - "name": "index operator class matches the search operator", - "passed": true, - "notes": "function operators: <#>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" - }, - { - "name": "user A search returns only own sections, best match first", - "passed": true - }, - { - "name": "user B search returns only own sections, best match first", - "passed": true - }, - { - "name": "user A reads only own sections through the API", - "passed": true - }, - { - "name": "user A reads only own documents through the API", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"pgvector semantic search match documents RLS HNSW vector extension Edge Functions\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#custom-roles-missing-passwords", + "title": "Custom roles missing passwords" + }, { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#legacy-studio-configuration", + "title": "Legacy Studio configuration" }, { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#connection-refused", + "title": "Connection refused" }, { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#before-you-begin", + "title": "Before you begin" }, { - "url": "https://supabase.com/docs/guides/ai/vector-indexes", - "title": "Vector indexes" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#step-1-get-your-platform-connection-string", + "title": "Step 1: Get your platform connection string" }, { - "url": "https://supabase.com/docs/guides/ai/going-to-prod", - "title": "Going to Production" - } - ], - "resultChars": 52858 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Edge Functions gte-small vector dimensions 384 match_document_sections\", limit: 8) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#step-2-back-up-your-platform-database", + "title": "Step 2: Back up your platform database" + }, { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#step-3-prepare-your-self-hosted-instance", + "title": "Step 3: Prepare your self-hosted instance" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#step-4-restore-to-your-self-hosted-database", + "title": "Step 4: Restore to your self-hosted database" }, { - "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", - "title": "Choosing your Compute Add-on" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#step-5-verify-the-restore", + "title": "Step 5: Verify the restore" }, { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#whats-included-in-the-restore-and-whats-not", + "title": "What's included in the restore and what's not" }, { - "url": "https://supabase.com/docs/guides/functions/recursive-functions", - "title": "Recursive / Nested Function Calls" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https", + "title": "Configure Reverse Proxy and HTTPS" }, { - "url": "https://supabase.com/docs/guides/ai/langchain", - "title": "LangChain" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https#websocket-connection-failed", + "title": "WebSocket connection failed" }, { - "url": "https://supabase.com/docs/guides/functions/architecture", - "title": "Edge Functions Architecture" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https#additional-resources", + "title": "Additional resources" }, { - "url": "https://supabase.com/docs/guides/storage/vector/working-with-indexes", - "title": "Working with Vector Indexes" - } - ], - "resultChars": 110370 - }, - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "pages": [ + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https#err_cert_authority_invalid", + "title": "ERR_CERT_AUTHORITY_INVALID" + }, { - "url": "https://supabase.com/changelog.md" - } - ] - }, - { - "source": "web_search", - "query": "site:supabase.com/changelog.md Supabase changelog pgvector Edge Functions semantic search", - "pages": [] - } - ] - }, - "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", - "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-vectors-001-rag-with-permissions.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "deploy-database-001-prometheus-metrics", - "stage": "deploy", - "product": [ - "database" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "passed": true, - "checks": [ - { - "name": "preserved existing app scrape job", - "passed": true - }, - { - "name": "configured the Supabase Metrics API scrape correctly", - "passed": true, - "judgeNotes": "Prometheus preserves the app job and adds a deployable Supabase scrape over HTTPS to /customer/v1/privileged/metrics for evalshostedprojectxy.supabase.co using basic_auth with password_file. docker-compose wires the matching password file via a Compose secret." - }, - { - "name": "documented live deployment and verification steps", - "passed": true, - "judgeNotes": "README includes creating a Supabase Secret API key, storing it in the Compose secret file path, recreating the stack, and verifying via Prometheus targets plus a direct metrics API curl check." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Prometheus metrics endpoint customer v1 privileged metrics service_role basic auth hosted project\", limit: 5) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https#mixed-content-warnings", + "title": "Mixed content warnings" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https#oauth-callback-url-mismatch", + "title": "OAuth callback URL mismatch" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https#step-3-update-configuration-variables", + "title": "Step 3: Update configuration variables" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https#step-4-restart-and-verify", + "title": "Step 4: Restart and verify" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https#troubleshooting", + "title": "Troubleshooting" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https#certificate-not-issued", + "title": "Certificate not issued" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https#step-2-start-the-reverse-proxy", + "title": "Step 2: Start the reverse proxy" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https#step-3-verify-https-connection", + "title": "Step 3: Verify HTTPS connection" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https#self-signed-certificates-development-only", + "title": "Self-signed certificates (development only)" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https#step-1-generate-a-self-signed-certificate", + "title": "Step 1: Generate a self-signed certificate" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https#step-2-configure-kong-for-ssl", + "title": "Step 2: Configure Kong for SSL" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https#before-you-begin", + "title": "Before you begin" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https#set-up-https", + "title": "Set up HTTPS" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https#step-1-update-environment-variables", + "title": "Step 1: Update environment variables" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17", + "title": "Upgrade to Postgres 17" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#pgsodium--supabase-vault-errors", + "title": "pgsodium / Supabase Vault errors" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#services-fail-to-connect-after-upgrade", + "title": "Services fail to connect after upgrade" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#postgres-17-fails-to-start-with-a-leftover-db-config-volume", + "title": "Postgres 17 fails to start with a leftover db-config volume" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#disk-space-issues-during-upgrade", + "title": "Disk space issues during upgrade" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#restoring-from-a-manual-backup", + "title": "Restoring from a manual backup" + }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#before-you-begin", + "title": "Before you begin" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#new-deployment-with-postgres-17", + "title": "New deployment with Postgres 17" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#upgrade-an-existing-postgres-15-deployment", + "title": "Upgrade an existing Postgres 15 deployment" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" - } - ], - "resultChars": 20081 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i 'metrics|prometheus|breaking-change' | head -80\"", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#what-the-upgrade-does", + "title": "What the upgrade does" + }, { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 3574 - } - ] - }, - "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", - "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/deploy-database-001-prometheus-metrics.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "deploy-functions-001-edge-function-secrets", - "stage": "deploy", - "product": [ - "edge-functions" - ], - "topic": [ - "security" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "WEATHER_API_KEY is set as a Function secret on the project", - "passed": true - }, - { - "name": "the weather function is deployed to the project", - "passed": true, - "notes": "status ACTIVE" - }, - { - "name": "the weather function reads WEATHER_API_KEY from the environment", - "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." - }, - { - "name": "WEATHER_API_KEY value is not committed to the repo", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions secrets environment variables WEATHER_API_KEY deploy CLI invoke CORS\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#create-a-backup", + "title": "Create a backup" + }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#requirements", + "title": "Requirements" }, { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#extensions-removed-in-postgres-17", + "title": "Extensions removed in Postgres 17" }, { - "url": "https://supabase.com/docs/guides/functions/development-environment", - "title": "Development Environment" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#run-the-upgrade", + "title": "Run the upgrade" }, { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting", - "title": "Troubleshooting" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#after-the-upgrade", + "title": "After the upgrade" }, { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#rollback", + "title": "Rollback" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - } - ], - "resultChars": 64693 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Edge Functions deploy secrets set env-file Deno.env.get invoke no verify jwt browser CORS\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#custom-postgres-configuration", + "title": "Custom Postgres configuration" + }, { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#upgrade-process-details", + "title": "Upgrade process details" }, { - "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", - "title": "Getting Started with Edge Functions (Dashboard)" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#troubleshooting", + "title": "Troubleshooting" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", - "title": "Inspecting edge function environment variables" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#pg_upgrade-fails-with-replication-slot-errors", + "title": "pg_upgrade fails with replication slot errors" }, { - "url": "https://supabase.com/docs/guides/ai/examples/openai", - "title": "Generating OpenAI GPT3 completions" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#test-with-the-aws-cli", + "title": "Test with the AWS CLI" }, { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#enable-the-s3-protocol-endpoint", + "title": "Enable the S3 protocol endpoint" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#test-with-rclone", + "title": "Test with rclone" }, { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#how-to-configure-an-s3-backend", + "title": "How to configure an S3 backend" }, { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#using-rustfs", + "title": "Using RustFS" }, { - "url": "https://supabase.com/docs/guides/functions/quickstart", - "title": "Getting Started with Edge Functions" - } - ], - "resultChars": 82037 - } - ] - }, - "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", - "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6/deploy-functions-001-edge-function-secrets.json" - }, - { - "experiment": "codex-gpt-5.6", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "deploy-self-hosting-001-docker-compose", - "stage": "deploy", - "product": [ - "database", - "auth", - "storage" - ], - "topic": [ - "self-hosting" - ], - "suite": "benchmark", - "interface": "cli", - "passed": true, - "checks": [ - { - "name": "cloned the self-host stack (docker-compose.yml + volumes/db)", - "passed": true - }, - { - "name": "didn't conflate with the CLI (no supabase/config.toml in the stack)", - "passed": true - }, - { - "name": "secrets rotated off the shipped defaults", - "passed": true - }, - { - "name": "ANON_KEY and SERVICE_ROLE_KEY are HS256 JWTs signed by JWT_SECRET", - "passed": true - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"self-hosting Docker docker-compose .env secrets JWT keys Vault logflare pooler official\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#using-minio", + "title": "Using MinIO" + }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#using-aws-s3", + "title": "Using AWS S3" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", - "title": "Configure Social Login (OAuth) Providers" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#s3-compatible-providers", + "title": "S3-compatible providers" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#verify", + "title": "Verify" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#session-token", + "title": "Session token" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#troubleshooting", + "title": "Troubleshooting" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#signature-mismatch-errors", + "title": "Signature mismatch errors" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-phone-mfa", - "title": "Configure Phone Login & MFA" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#tus-upload-errors-on-cloudflare-r2", + "title": "TUS upload errors on Cloudflare R2" }, { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates", - "title": "Custom Email Templates" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#permission-denied-on-uploads", + "title": "Permission denied on uploads" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#upload-urls-point-to-localhost", + "title": "Upload URLs point to localhost" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" - } - ], - "resultChars": 131466 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"set -e\nprintf '%s\\\\n' '--- changelog tags relevant to self-hosting ---'\ncurl -fsSL https://supabase.com/changelog.md | rg -i -n -m 30 'breaking-change|self-host|docker|auth key|api key' || true\nrm -rf /tmp/supabase-upstream\ngit clone --depth 1 --filter=blob:none --sparse https://github.com/supabase/supabase.git /tmp/supabase-upstream >/tmp/supabase-clone.log 2>&1\ngit -C /tmp/supabase-upstream sparse-checkout set docker\nprintf '%s\\\\n' '--- upstream revision ---'\ngit -C /tmp/supabase-upstream rev-parse HEAD\ngit -C /tmp/supabase-upstream log -1 --format='%cI %s'\nprintf '%s\\\\n' '--- docker files ---'\nfind /tmp/supabase-upstream/docker -maxdepth 3 -type f | sed 's#/tmp/supabase-upstream/docker/##' | sort | sed -n '1,240p'\nprintf '%s\\\\n' '--- docker changelog head ---'\nsed -n '1,220p' /tmp/supabase-upstream/docker/CHANGELOG.md\nprintf '%s\\\\n' '--- env variables ---'\nsed -n '1,260p' /tmp/supabase-upstream/docker/.env.example\"", - "pages": [ + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#additional-resources", + "title": "Additional resources" + }, { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#permission-denied-on-the-data-directory" } ], - "resultChars": 29108 + "resultChars": 459557 } ] }, @@ -14810,7 +15743,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "The answer identifies the soft-delete-only root cause, implements deletion of auth.users and RLS checks to close the data path for stale JWTs, consistently explains that JWTs remain cryptographically valid until exp while data access is blocked by the implemented RLS, and correctly distinguishes publishable/frontend/RLS keys from secret/backend/bypass-RLS keys." + "judgeNotes": "The answer identifies the soft-delete-only root cause, implements deletion of auth.users cascading sessions/refresh tokens, adds active-session RLS to close the Data API window for audited tables, explains JWTs remain locally valid until exp with caveat for paths missing the check, and correctly distinguishes publishable vs secret keys including secret bypassing RLS and never client-side." } ], "skills": { @@ -14827,93 +15760,102 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"delete user existing access token session revoke refresh token admin deleteUser signOut scope global\", limit: 8) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"delete user does not invalidate access token sessions sign out revoke sessions JWT expiry delete account\", limit: 8) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/auth-signout" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/auth/signout", - "title": "Signing out" + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" + "url": "https://supabase.com/docs/reference/swift/auth-admin-signout", + "title": "signOut()" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" + "url": "https://supabase.com/docs/reference/javascript/auth-signout" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-signout" + "url": "https://supabase.com/docs/guides/platform/sso/testing-best-practices", + "title": "SSO Testing and Best Practices" }, { - "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" }, { "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", "title": "Multiple SSO Providers" }, { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" } ], - "resultChars": 68746 + "resultChars": 106161 }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"cat .agents/skills/supabase-postgres-best-practices/references/security-privileges.md; printf '\\\\n---RLS---\\\\n'; cat .agents/skills/supabase-postgres-best-practices/references/security-rls-basics.md; printf '\\\\n---CHANGELOG MATCHES---\\\\n'; curl -fsSL https://supabase.com/changelog.md | rg -i -n 'breaking-change|auth|jwt|api key|publishable|secret' | head -80\"", + "source": "web_search", + "query": "https://supabase.com/changelog.md", "pages": [ { "url": "https://supabase.com/changelog.md" } - ], - "resultChars": 14841 + ] + }, + { + "source": "web_search", + "query": "site:supabase.com/changelog auth sessions API keys publishable secret user deletion access token", + "pages": [] }, { "source": "search_docs", - "query": "query { searchDocs(query: \"publishable key secret key RLS bypass service_role anon new API keys sb_publishable sb_secret\", limit: 10) { nodes { ... on Guide { title href content } ... on ManagementApiReference { title href content } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"delete account database function auth.users security definer delete current user existing JWT session_id RLS auth.sessions\", limit: 10) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/auth/auth-anonymous", + "title": "Anonymous Sign-Ins" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" }, { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" + "url": "https://supabase.com/docs/guides/auth/sessions", + "title": "User sessions" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" + "url": "https://supabase.com/docs/guides/storage/management/delete-objects", + "title": "Delete Objects" }, { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/storage/security/ownership", + "title": "Ownership" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" } ], - "resultChars": 137700 + "resultChars": 121871 } ] }, @@ -14968,7 +15910,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Identified orders missing from supabase_realtime despite SUBSCRIBED channel, fixed with ALTER PUBLICATION ADD TABLE public.orders, preserved courier_locations and RLS/policies." + "judgeNotes": "The assistant correctly identified the root cause as orders missing from the supabase_realtime publication despite channel SUBSCRIBED, added only public.orders to the existing publication, verified courier_locations remained published, and did not weaken RLS/policies or blame client/RLS/networking." } ], "skills": { @@ -14985,17 +15927,17 @@ "calls": [ { "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- files ---'; rg --files -g '\"'!node_modules'\"' -g '\"'!dist'\"' | sed -n '1,240p'; printf '%s\\\\n' '--- realtime/order refs ---'; rg -n \\\"postgres_changes|orders|courier|supabase_realtime|publication\\\" -g '\"'!node_modules'\"' -g '\"'!dist'\"' . || true; printf '%s\\\\n' '--- changelog realtime refs ---'; curl -fsSL https://supabase.com/changelog.md | rg -n -i \\\"realtime|postgres_changes|publication\\\" | head -40 || true\"", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- files ---'; rg --files -g '\"'!*node_modules*'\"' | sed -n '1,200p'; printf '%s\\\\n' '--- realtime references ---'; rg -n \\\"postgres_changes|courier|orders|supabase_realtime|publication\\\" -g '\"'!*node_modules*'\"' . | sed -n '1,260p'; printf '%s\\\\n' '--- changelog realtime mentions ---'; curl -fsSL https://supabase.com/changelog.md | rg -n -i \\\"realtime|postgres_changes|breaking-change\\\" | head -80\"", "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 28635 + "resultChars": 32335 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Postgres Changes enable table supabase_realtime publication add table postgres_changes RLS\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"Postgres Changes subscribe table add to supabase_realtime publication postgres_changes\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ { @@ -15006,20 +15948,20 @@ "url": "https://supabase.com/docs/guides/realtime/postgres-changes", "title": "Postgres Changes" }, - { - "url": "https://supabase.com/docs/guides/realtime/authorization", - "title": "Realtime Authorization" - }, { "url": "https://supabase.com/docs/guides/realtime/benchmarks", "title": "Benchmarks" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/realtime/protocol", + "title": "Realtime Protocol" + }, + { + "url": "https://supabase.com/docs/guides/realtime/getting_started", + "title": "Getting Started with Realtime" } ], - "resultChars": 92624 + "resultChars": 126779 } ] }, @@ -15052,17 +15994,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "The assistant explicitly identified `image-transform` as affected and described eight recurring 503 responses across 07:00–12:00 UTC on 2026-04-28, matching the required pattern." + "judgeNotes": "Identified image-transform and described 8 gateway-level HTTP 503s recurring through the morning of 2026-04-28, with timing/pattern and distinction from other issues." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "The assistant explicitly attributes the recurring image-transform 503s to the API Gateway/Edge Runtime/platform layer before invocation, not function code. This is grounded in valid observations: the 503s reached the gateway but had no Edge Function execution records, nearby invocations succeeded, and it distinguishes the unrelated avatar-upload 500 as an application-level error. It also advises not to redeploy or modify function/storage/database config." + "judgeNotes": "The assistant clearly attributes the recurring image-transform 503s to the gateway/platform layer before the Edge runtime, not the function code. This is grounded in valid observations: no corresponding Edge Function execution logs for failed requests, nearby successful 200s on the same deployment, and distinction from the isolated avatar-upload function-level 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended concrete next steps, including opening a Supabase support ticket with the project, UTC window, and request IDs, plus capturing specific failure metadata and adding retries." + "judgeNotes": "The assistant recommended concrete next steps: opening a Supabase support ticket with project/function/deployment/time details, correlating gateway request IDs with infrastructure logs, adding retries, and monitoring gateway responses separately from function executions." } ], "skills": { @@ -15071,82 +16013,38 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { "calls": [ - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ] - }, - { - "source": "web_search", - "query": "site:supabase.com/changelog.md Supabase changelog storage April 2026", - "pages": [] - }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions 503 gateway before invocation logs troubleshooting retry transient errors\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Edge Functions 503 gateway error function not invoked logs troubleshooting\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" - }, { "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-503-response", "title": "Edge Function 503 error response" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-504-error-response", - "title": "Edge Function 504 error response" - }, - { - "url": "https://supabase.com/docs/guides/functions/recursive-functions", - "title": "Recursive / Nested Function Calls" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - } - ], - "resultChars": 43053 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"function_edge_logs execution_id function_id internal_failure 503\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-503-response", - "title": "Edge Function 503 error response" + "url": "https://supabase.com/docs/guides/functions/status-codes", + "title": "Status codes" }, { "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-504-error-response", "title": "Edge Function 504 error response" }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-404-error-response", + "title": "Edge Function 404 error response" + }, { "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-500-error-response", "title": "Edge Function 500 error response" } ], - "resultChars": 32009 - }, - { - "source": "web_search", - "query": "https://supabase.com/docs/guides/troubleshooting/edge-function-503-response", - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-503-response" - } - ] + "resultChars": 41055 } ] }, @@ -15206,7 +16104,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Diagnosed RLS enabled with no policies causing deny-all Data API behavior, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts. No permissive/public/anon policies or RLS disabling." + "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies as deny-all for Data API access, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() using USING and WITH CHECK. No permissive/anon policies or RLS disabling." } ], "skills": { @@ -15223,17 +16121,17 @@ "calls": [ { "source": "shell_fetch", - "query": "/bin/bash -lc \"cat .agents/skills/supabase-postgres-best-practices/references/security-rls-basics.md; printf '\\\\n---PERF---\\\\n'; cat .agents/skills/supabase-postgres-best-practices/references/security-rls-performance.md; printf '\\\\n---PRIVILEGES---\\\\n'; cat .agents/skills/supabase-postgres-best-practices/references/security-privileges.md; printf '\\\\n---CHANGELOG MATCHES---\\\\n'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|data api|row level|rls' | head -120\"", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i \\\"breaking|data api|row level|rls|grant\\\" | head -80; printf '\\\\n--- relevant RLS guidance ---\\\\n'; cat .claude/skills/supabase-postgres-best-practices/references/security-rls-performance.md 2>/dev/null || true; cat .claude/skills/supabase-postgres-best-practices/references/security-rls.md 2>/dev/null || true\"", "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 8535 + "resultChars": 16947 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Data API table grants authenticated RLS select insert policy auth.uid bookmarks\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Data API table grants authenticated RLS policies auth.uid select insert\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -15241,23 +16139,23 @@ "title": "Securing your API" }, { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" }, { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" }, { - "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", - "title": "Storage Helper Functions" + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" } ], - "resultChars": 51520 + "resultChars": 90111 } ] }, @@ -15310,7 +16208,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "PASS. The agent added the missing local bio migration file `supabase/migrations/20240115000000_add_profile_bio.sql`, which reconciled the orphan remote history entry for CLI pushes. The pending avatar migration was then applied through a real `supabase db push` in command #10, whose output includes `Finished supabase db push` (and #11 confirms remote history contains `20240220000000 | add_avatar_url`). No prohibited direct-SQL mutation or prepared-statement workaround was seen." + "judgeNotes": "Avatar migration was applied through Supabase CLI with `supabase db push --db-url \"$DB_URL\" --yes`, which output `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration history was reconciled by adding the missing local migration file `supabase/migrations/20240115000000_add_profile_bio.sql`, after which the CLI push succeeded with the remote history showing all three versions. No disallowed direct SQL mutation or prepared-statement workaround was seen." } ], "skills": { @@ -15327,81 +16225,31 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI db push migration history repair linked project deploy migrations\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase CLI db push migration history repair remote migration versions duplicate column\", limit: 5) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, { "url": "https://supabase.com/docs/reference/cli/supabase-migration-repair", "title": "Repair the migration history table" }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-db-push", - "title": "Push new migrations to the remote database" - }, - { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" - } - ], - "resultChars": 61657 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"set -o pipefail\nprintf '%s\\\\n' '--- CHANGELOG BREAKING CHANGES ---'\ncurl -fsSL https://supabase.com/changelog.md | rg -n -i -C 2 'breaking-change|migration|db push' | head -120 || true\nprintf '%s\\\\n' '--- GIT STATUS ---'\ngit status --short --branch 2>&1 || true\nprintf '%s\\\\n' '--- SUPABASE FILES ---'\nfind supabase -maxdepth 4 -type f -print | sort\nprintf '%s\\\\n' '--- CONFIG ---'\nsed -n '1,260p' supabase/config.toml 2>/dev/null || true\nprintf '%s\\\\n' '--- MIGRATIONS ---'\nfor f in supabase/migrations/*.sql; do echo \\\"===== \"'$f\"; sed -n '\"'1,260p' \\\"\"'$f\"; done\nprintf '\"'%s\\\\n' '--- LOCAL LINK METADATA ---'\nfind supabase/.temp -maxdepth 2 -type f -print -exec sh -c 'echo \\\"--- \"'$1\"; cat \"$1\"'\"' _ {} \\\\; 2>/dev/null || true\nprintf '%s\\\\n' '--- CLI / ENV NAMES ---'\nsupabase --version\nprintf 'SUPABASE vars: '; env | cut -d= -f1 | rg '\"'^(SUPABASE|POSTGRES|PG)'\"' | sort | tr '\\\\n' ' '; echo\nprintf '%s\\\\n' '--- PROFILE ---'\ncat .supabase-eval-profile.yaml\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 478 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase migration history remote migration versions not found local migration repair db pull migration fetch\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, { "url": "https://supabase.com/docs/guides/deployment/database-migrations", "title": "Database Migrations" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/new-branch-doesnt-copy-database", - "title": "Why are my Supabase branches empty?" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-migration-repair", - "title": "Repair the migration history table" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/branch-in-migrations-failed-status", - "title": "Troubleshooting MIGRATIONS_FAILED: missing tables or an incomplete schema on your branch" + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" }, { "url": "https://supabase.com/docs/reference/cli/supabase-db-push", "title": "Push new migrations to the remote database" }, { - "url": "https://supabase.com/docs/reference/cli/supabase-migration-fetch", - "title": "Fetch migration files from history table" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-db-pull", - "title": "Pull schema from the remote database" + "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", + "title": "Backup and Restore using the CLI" } ], - "resultChars": 58381 + "resultChars": 84771 } ] }, @@ -15468,34 +16316,41 @@ "calls": [ { "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | rg -n -i \"breaking-change|postgres|index|database\" | head -40'", - "hasContent": true, + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- files ---'; rg --files -g 'AGENTS.md' -g 'supabase/**' -g '*.sql' -g 'package.json' | sed -n '1,200p'; printf '%s\\\\n' '--- relevant best practices ---'; cat .agents/skills/supabase-postgres-best-practices/references/query-missing-indexes.md; printf '\\\\n--- partial ---\\\\n'; cat .agents/skills/supabase-postgres-best-practices/references/query-partial-indexes.md; printf '\\\\n--- composite ---\\\\n'; if [ -f .agents/skills/supabase-postgres-best-practices/references/query-composite-indexes.md ]; then cat .agents/skills/supabase-postgres-best-practices/references/query-composite-indexes.md; else rg -l 'composite|column order|ORDER BY' .agents/skills/supabase-postgres-best-practices/references/query-* | head -10; fi; printf '\\\\n--- changelog scan ---\\\\n'; curl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|postgres|index|performance' | head -30\"", "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 7495 + "resultChars": 10111 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Postgres query optimization composite index ORDER BY WHERE EXPLAIN\", limit: 3) { nodes { ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"database query optimization EXPLAIN ANALYZE composite index ORDER BY LIMIT pg_stat_statements\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/database/orioledb", + "title": "OrioleDB Overview" + }, + { + "url": "https://supabase.com/docs/guides/database/inspect", + "title": "Database debugging and monitoring" + }, { "url": "https://supabase.com/docs/guides/database/query-optimization", "title": "Query Optimization" }, { - "url": "https://supabase.com/docs/guides/database/orioledb", - "title": "OrioleDB Overview" + "url": "https://supabase.com/docs/guides/troubleshooting/running-explain-analyze-on-functions", + "title": "Running EXPLAIN ANALYZE on functions" }, { - "url": "https://supabase.com/docs/guides/realtime/reports", - "title": "Realtime Reports" + "url": "https://supabase.com/docs/guides/troubleshooting/understanding-postgresql-explain-output-Un9dqX", + "title": "Understanding Postgres EXPLAIN Output" } ], - "resultChars": 63315 + "resultChars": 43518 } ] }, @@ -15577,42 +16432,46 @@ "docs": { "calls": [ { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- files ---'; rg --files -g 'AGENTS.md' -g 'supabase/**' -g 'package.json' -g '*.sql' | sed -n '1,240p'; printf '%s\\\\n' '--- changelog security/rls mentions ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -n -m 20 'breaking-change|row level|rls|policy|security'\"", + "source": "web_search", + "query": "https://supabase.com/changelog.md", "pages": [ { "url": "https://supabase.com/changelog.md" } - ], - "resultChars": 4349 + ] + }, + { + "source": "web_search", + "query": "site:supabase.com/changelog.md Supabase changelog breaking change RLS", + "pages": [] }, { "source": "search_docs", - "query": "query { searchDocs(query: \"row level security multi tenant workspace membership policies auth uid index policy performance\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"row level security workspace membership multi tenant policies auth.uid exists\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" }, { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" + "url": "https://supabase.com/docs/guides/realtime/authorization", + "title": "Realtime Authorization" }, { - "url": "https://supabase.com/docs/guides/database/database-advisors", - "title": "Performance and Security Advisors" + "url": "https://supabase.com/docs/guides/realtime/getting_started", + "title": "Getting Started with Realtime" }, { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" + "url": "https://supabase.com/docs/guides/auth/auth-mfa", + "title": "Multi-Factor Authentication" } ], - "resultChars": 48168 + "resultChars": 90123 } ] }, @@ -15653,7 +16512,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 14e42a99-dd08-453c-a361-a8b7871b40cc, signUp returned {\"userId\":\"14e42a99-dd08-453c-a361-a8b7871b40cc\"}" + "notes": "db user 22f1d341-284e-4abc-91a3-f943913c80fc, signUp returned {\"userId\":\"22f1d341-284e-4abc-91a3-f943913c80fc\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -15668,7 +16527,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"14e42a99-dd08-453c-a361-a8b7871b40cc\"}" + "notes": "{\"userId\":\"22f1d341-284e-4abc-91a3-f943913c80fc\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -15694,13 +16553,33 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js createClient signUp user metadata signInWithPassword getSession select single profiles\", limit: 8) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"supabase-js createClient signUp user metadata signInWithPassword auth getUser select single RLS\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, { "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", "title": "Migrate from Auth0 to Supabase Auth" }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "title": "Why is my service role key client getting RLS errors or not returning data?" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-google", + "title": "Login with Google" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-slack", + "title": "Login with Slack" + }, { "url": "https://supabase.com/docs/guides/auth/social-login/auth-linkedin", "title": "Login with LinkedIn" @@ -15708,29 +16587,38 @@ { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", "title": "Configure SAML SSO" + } + ], + "resultChars": 151077 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"JavaScript signUp email password options data user metadata signInWithPassword\", limit: 6) { nodes { __typename title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", - "title": "Login with Figma" + "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-google", - "title": "Login with Google" + "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", + "title": "signInWithPassword()" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-gitlab", - "title": "Login with GitLab" + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-slack", - "title": "Login with Slack" + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpasskey" }, { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" + "url": "https://supabase.com/docs/reference/swift/auth-signinwithpasskey", + "title": "signInWithPasskey()" } ], - "resultChars": 166581 + "resultChars": 50844 } ] }, @@ -15802,28 +16690,28 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Row Level Security authenticated users SELECT policy anon no rows migrations seed local development\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Row Level Security authenticated role select policy anon signed in users auth.uid migrations seed local development\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { "url": "https://supabase.com/docs/guides/local-development/cli-workflows", "title": "Local development workflow" }, { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { "url": "https://supabase.com/docs/guides/auth/auth-mfa", "title": "Multi-Factor Authentication" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" + "url": "https://supabase.com/docs/guides/auth/auth-anonymous", + "title": "Anonymous Sign-Ins" } ], "resultChars": 95821 @@ -15882,7 +16770,7 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI create migration alter table add column local database\", limit: 3) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase CLI create migration alter table add column local database\", limit: 3) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } } } }", "hasContent": true, "pages": [ { @@ -15945,7 +16833,7 @@ { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 6) from the queue" + "notes": "function removed the seeded message (id 4) from the queue" } ], "skills": { @@ -15956,75 +16844,75 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Queues pgmq create queue send messages Edge Function read delete pgmq_public cron schedule every minute\", limit: 8) { nodes { __typename title href content } } }", + "query": "query { searchDocs(query: \"Supabase Queues pgmq cron pg_cron send message read delete Edge Function local\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" - }, { "url": "https://supabase.com/docs/guides/functions/schedule-functions", "title": "Scheduling Edge Functions" }, { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "title": "Consuming Supabase Queue Messages with Edge Functions" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" }, { "url": "https://supabase.com/docs/guides/cron", "title": "Cron" }, { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" }, { - "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues", - "title": "Expose Queues for local and self-hosted Supabase" + "url": "https://supabase.com/docs/guides/cron/quickstart", + "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", + "title": "Testing and linting" } ], - "resultChars": 39939 + "resultChars": 68190 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Queues Quickstart pgmq.create cron.schedule SQL schedule database jobs\", limit: 6) { nodes { __typename title href content } } }", + "query": "query { searchDocs(query: \"Supabase Queues create queue SQL pgmq.create cron.schedule enqueue message every minute\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/cron/install", - "title": "Install" + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" }, { - "url": "https://supabase.com/docs/guides/cron/quickstart", - "title": "Quickstart" + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" }, { "url": "https://supabase.com/docs/guides/cron", "title": "Cron" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", - "title": "pg_cron debugging guide" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" }, { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" + "url": "https://supabase.com/docs/guides/cron/install", + "title": "Install" }, { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" } ], - "resultChars": 42534 + "resultChars": 52533 } ] }, @@ -16087,12 +16975,211 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"supabase javascript client select nested relationships aggregate query\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/joins-and-nesting", + "title": "Querying Joins and Nested tables" + }, + { + "url": "https://supabase.com/docs/guides/api/sql-to-api", + "title": "Converting SQL to JavaScript API" + }, + { + "url": "https://supabase.com/docs/guides/api/automatic-retries-in-supabase-js", + "title": "How to do automatic retries with `supabase-js`" + }, + { + "url": "https://supabase.com/docs/reference/javascript/schema" + }, + { + "url": "https://supabase.com/docs/guides/api/creating-routes", + "title": "Creating API Routes" + } + ], + "resultChars": 37432 + } + ] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6-no-skills/build-dataapi-001-relational-report.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"supabase javascript client select nested relationships service role Node.js\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" + }, + { + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", + "title": "Serverless Drivers" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-listclients" + }, + { + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" + } + ], + "resultChars": 19738 + } + ] + }, + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6-no-skills/build-dataapi-002-restock-alert-report.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "build-database-001-migrate-postgres-to-supabase", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "all 3 tables exist (teams, members, tasks)", + "passed": true + }, + { + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true + }, + { + "name": "foreign key constraints survived the restore", + "passed": true + }, + { + "name": "tasks_team_status_idx index survived the restore", + "passed": true + }, + { + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI local development restore pg_dump custom format existing database\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", + "title": "Migrate from Postgres to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" + }, + { + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", + "title": "Backup and Restore using the CLI" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", + "title": "Migrate from Vercel Postgres to Supabase" + } + ], + "resultChars": 75030 + } + ] }, - "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", - "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.6-no-skills/build-dataapi-001-relational-report.json" + "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", + "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6-no-skills/build-database-001-migrate-postgres-to-supabase.json" }, { "experiment": "codex-gpt-5.6-no-skills", @@ -16103,44 +17190,46 @@ "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "build-dataapi-002-restock-alert-report", + "eval": "build-functions-004-service-role-bypass", "stage": "build", "product": [ - "data-api", + "edge-functions", + "auth", "database" ], "topic": [ + "rls", + "security", "sdk" ], "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, + "interface": "mcp", + "passed": true, "checks": [ { - "name": "report runs and prints JSON", + "name": "rejects missing auth", "passed": true, - "notes": "exit 0" + "notes": "status=401" }, { - "name": "alerts match the database (below threshold, sorted)", + "name": "user A reads own note", "passed": true, - "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + "notes": "status=200" }, { - "name": "tables stay locked down (publishable key reads nothing)", + "name": "reads only with the caller's JWT", "passed": true, - "notes": "publishable read errored: permission denied for table inventory" + "notes": "bearer_tokens=2, all_match=true" }, { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + "name": "user A cannot force-read user B note", + "passed": true, + "notes": "status=200" }, { - "name": "report queries via the Data API, not raw SQL", + "name": "user B cannot force-read user A note", "passed": true, - "notes": "no psql / raw Postgres driver usage found" + "notes": "status=200" } ], "skills": { @@ -16151,38 +17240,34 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript supabase-js select foreign tables nested relationships service role Node backend\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Edge Functions authenticate user Authorization header createClient RLS anon key getUser\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/ai/engineering-for-scale", - "title": "Engineering for Scale" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" }, { - "url": "https://supabase.com/docs/guides/database/joins-and-nesting", - "title": "Querying Joins and Nested tables" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" }, { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" } ], - "resultChars": 45250 + "resultChars": 65550 } ] }, - "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", - "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.6-no-skills/build-dataapi-002-restock-alert-report.json" + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6-no-skills/build-functions-004-service-role-bypass.json" }, { "experiment": "codex-gpt-5.6-no-skills", @@ -16193,37 +17278,67 @@ "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "build-database-001-migrate-postgres-to-supabase", + "eval": "build-functions-005-dual-auth-user-secret", "stage": "build", "product": [ + "edge-functions", + "auth", "database" ], "topic": [ - "migrations" + "sdk", + "rls", + "security" ], "suite": "benchmark", "interface": "cli", + "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true + "name": "seed rows present", + "passed": true, + "notes": "found 2/2 seeded rows" }, { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true + "name": "rejects request with no credentials", + "passed": true, + "notes": "status 401: {\"error\":\"Unauthorized\"}" }, { - "name": "foreign key constraints survived the restore", - "passed": true + "name": "user with JWT reads only their own rows", + "passed": true, + "notes": "status 200: [{\"user_id\":\"f9530622-f486-45b6-9acd-4043e7be62b9\",\"metric\":\"steps_a_mssyypuu\",\"value\":111}]" }, { - "name": "tasks_team_status_idx index survived the restore", - "passed": true + "name": "user cannot read another user's rows by passing user_id", + "passed": true, + "notes": "status 200: [{\"user_id\":\"f9530622-f486-45b6-9acd-4043e7be62b9\",\"metric\":\"steps_a_mssyypuu\",\"value\":111}]" }, { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true + "name": "service key bypasses RLS to read the target user's rows", + "passed": true, + "notes": "status 200: [{\"user_id\":\"570315c4-ae6e-4770-a560-879c88f111de\",\"metric\":\"steps_b_mssyypuu\",\"value\":222}]" + }, + { + "name": "non-service key is not granted service access", + "passed": true, + "notes": "status 401: {\"error\":\"Unauthorized\"}" + }, + { + "name": "rejects an unverified (forged) user token", + "passed": true, + "notes": "status 401: {\"error\":\"Unauthorized\"}" + }, + { + "name": "a user token in the apikey slot is not treated as the service key", + "passed": true, + "notes": "status 401: {\"error\":\"Unauthorized\"}" + }, + { + "name": "implementation uses @supabase/server", + "passed": true, + "notes": "imports @supabase/server / withSupabase" } ], "skills": { @@ -16234,38 +17349,150 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI restore pg_dump custom format local database migrate existing Postgres\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase Edge Functions verify_jwt false config.toml auth getUser Authorization service role key apikey\", limit: 8) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/development-tips", + "title": "Development tips" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + }, + { + "url": "https://supabase.com/docs/guides/auth/jwts", + "title": "JSON Web Token (JWT)" + } + ], + "resultChars": 69113 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Function createClient Authorization header getUser JWT RLS user context Deno SUPABASE_URL SUPABASE_ANON_KEY\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", - "title": "Migrate from Postgres to Supabase" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/functions/websockets", + "title": "Handling WebSockets" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" + } + ], + "resultChars": 56199 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Functions default secrets SUPABASE_SERVICE_ROLE_KEY SUPABASE_SECRET_KEY environment variables\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + } + ], + "resultChars": 41809 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"@supabase/server withSupabase auth userClaims sub ctx.supabase TypeScript Edge Function\", limit: 8) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" + }, + { + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", + "title": "Which package to use" }, { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", - "title": "Migrate from Vercel Postgres to Supabase" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", + "title": "Build a User Management App with Angular" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/neon", - "title": "Migrate from Neon to Supabase" + "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", + "title": "Resumable WebSockets with Edge Functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", - "title": "Restore a Platform Project to Self-Hosted" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-react", + "title": "Build a User Management App with React" } ], - "resultChars": 42818 + "resultChars": 143630 } ] }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-database-001-migrate-postgres-to-supabase.json" + "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", + "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6-no-skills/build-functions-005-dual-auth-user-secret.json" }, { "experiment": "codex-gpt-5.6-no-skills", @@ -16276,16 +17503,14 @@ "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "build-functions-004-service-role-bypass", + "eval": "build-storage-001-private-bucket-access", "stage": "build", "product": [ - "edge-functions", - "auth", + "storage", "database" ], "topic": [ "rls", - "security", "sdk" ], "suite": "benchmark", @@ -16293,29 +17518,42 @@ "passed": true, "checks": [ { - "name": "rejects missing auth", - "passed": true, - "notes": "status=401" + "name": "bucket user-files exists", + "passed": true }, { - "name": "user A reads own note", - "passed": true, - "notes": "status=200" + "name": "bucket user-files is private", + "passed": true }, { - "name": "reads only with the caller's JWT", - "passed": true, - "notes": "bearer_tokens=2, all_match=true" + "name": "RLS still enabled on storage.objects", + "passed": true }, { - "name": "user A cannot force-read user B note", + "name": "user A lists only own files", "passed": true, - "notes": "status=200" + "notes": "saw: 01a0005e-3005-738e-8e17-309509d8ce53/receipt-alpha.pdf, 01a0005e-3005-738e-8e17-309509d8ce53/receipt-beta.pdf" }, { - "name": "user B cannot force-read user A note", + "name": "user B cannot read user A files", + "passed": true + }, + { + "name": "anon reads no files", + "passed": true + }, + { + "name": "user A can upload into own folder", + "passed": true + }, + { + "name": "user B cannot upload into user A folder", + "passed": true + }, + { + "name": "configured private per-user storage access", "passed": true, - "notes": "status=200" + "judgeNotes": "Meets requirements: private user-files bucket, authenticated owner-scoped SELECT and INSERT policies with WITH CHECK, no RLS disable or public access, and supabase-js createSignedUrl with expiry." } ], "skills": { @@ -16326,38 +17564,38 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions authenticate user Authorization header getUser createClient SUPABASE_ANON_KEY RLS\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase Storage RLS policies foldername auth.uid createSignedUrl private bucket\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" + "url": "https://supabase.com/docs/guides/storage/schema/helper-functions", + "title": "Storage Helper Functions" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", + "title": "Custom Roles" } ], - "resultChars": 32789 + "resultChars": 26660 } ] }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", + "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", + "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-functions-004-service-role-bypass.json" + "sourcePath": "codex-gpt-5.6-no-skills/build-storage-001-private-bucket-access.json" }, { "experiment": "codex-gpt-5.6-no-skills", @@ -16368,67 +17606,33 @@ "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "build-functions-005-dual-auth-user-secret", + "eval": "build-tests-001-rls-tenant-isolation", "stage": "build", "product": [ - "edge-functions", - "auth", "database" ], "topic": [ - "sdk", - "rls", - "security" + "tests", + "rls" ], "suite": "benchmark", "interface": "cli", - "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"7771b8ed-5e55-4cd4-af22-e43905cae0fe\",\"metric\":\"steps_a_msj0yta6\",\"value\":111}]" - }, - { - "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: [{\"user_id\":\"7771b8ed-5e55-4cd4-af22-e43905cae0fe\",\"metric\":\"steps_a_msj0yta6\",\"value\":111}]" - }, - { - "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: [{\"user_id\":\"7bc5ace7-c5be-42dc-ba44-d5537f012dc1\",\"metric\":\"steps_b_msj0yta6\",\"value\":222}]" - }, - { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "rejects an unverified (forged) user token", + "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" }, { - "name": "a user token in the apikey slot is not treated as the service key", + "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "notes": "6 passed, 0 failed" }, { - "name": "implementation uses @supabase/server", + "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "notes": "imports @supabase/server / withSupabase" + "judgeNotes": "The agent correctly identified `posts` as having broken tenant isolation, grounded it in pgTAP failures showing cross-tenant post reads, and did not blame `notes` or dismiss the tests." } ], "skills": { @@ -16439,145 +17643,38 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions authenticate user JWT Authorization header service role secret key apikey header verify_jwt false getClaims\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-getclaims" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-getclaims", - "title": "getClaims()" - } - ], - "resultChars": 46978 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"API keys publishable secret key apikey header Edge Functions secret key service_role authorization\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", - "title": "Getting Started with Edge Functions (Dashboard)" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" - } - ], - "resultChars": 177206 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"@supabase/server Edge Functions API key environment SUPABASE_SECRET_KEYS authenticate request\", limit: 10) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"database testing pgTAP row level security auth.uid tenant isolation Supabase CLI test db\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", + "title": "Testing and linting" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/local-development/testing/overview", + "title": "Testing Overview" }, { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/reference/cli/supabase-test-db", + "title": "Tests local database with pgTAP" }, { - "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration", - "title": "Build a Supabase Integration" + "url": "https://supabase.com/docs/guides/database/testing", + "title": "Testing Your Database" } ], - "resultChars": 90248 + "resultChars": 43099 } ] }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", + "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", + "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-functions-005-dual-auth-user-secret.json" + "sourcePath": "codex-gpt-5.6-no-skills/build-tests-001-rls-tenant-isolation.json" }, { "experiment": "codex-gpt-5.6-no-skills", @@ -16588,57 +17685,50 @@ "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "build-storage-001-private-bucket-access", + "eval": "build-vectors-001-rag-with-permissions", "stage": "build", "product": [ - "storage", - "database" + "database", + "vectors" ], "topic": [ - "rls", - "sdk" + "sql", + "rls" ], "suite": "benchmark", "interface": "mcp", "passed": true, "checks": [ { - "name": "bucket user-files exists", - "passed": true - }, - { - "name": "bucket user-files is private", - "passed": true + "name": "document_sections.embedding is vector(384)", + "passed": true, + "notes": "vector(384)" }, { - "name": "RLS still enabled on storage.objects", - "passed": true + "name": "HNSW index on the embedding column", + "passed": true, + "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" }, { - "name": "user A lists only own files", + "name": "index operator class matches the search operator", "passed": true, - "notes": "saw: 019fdc8d-04dc-730f-a0ac-4ed1bc5a28ce/receipt-alpha.pdf, 019fdc8d-04dc-730f-a0ac-4ed1bc5a28ce/receipt-beta.pdf" + "notes": "function operators: <#>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_ip_ops)" }, { - "name": "user B cannot read user A files", + "name": "user A search returns only own sections, best match first", "passed": true }, { - "name": "anon reads no files", + "name": "user B search returns only own sections, best match first", "passed": true }, { - "name": "user A can upload into own folder", + "name": "user A reads only own sections through the API", "passed": true }, { - "name": "user B cannot upload into user A folder", + "name": "user A reads only own documents through the API", "passed": true - }, - { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "Meets rubric: private user-files bucket (public=false), authenticated SELECT and INSERT policies scoped to bucket and auth.uid() path prefix with WITH CHECK for upload, no RLS disabling/permissive public policies, and supabase-js createSignedUrl with expiry for temporary sharing." } ], "skills": { @@ -16649,38 +17739,38 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Storage access control RLS policies storage.objects foldername auth.uid signed URL createSignedUrl upload download private bucket\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "query": "query { searchDocs(query: \"pgvector semantic search match_documents function RLS auth.uid security invoker gte-small 384\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" }, { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/guides/ai/hybrid-search", + "title": "Hybrid search" }, { - "url": "https://supabase.com/docs/guides/storage/schema/custom-roles", - "title": "Custom Roles" + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/guides/ai/langchain", + "title": "LangChain" }, { - "url": "https://supabase.com/docs/guides/security/product-security", - "title": "Secure configuration of Supabase products" + "url": "https://supabase.com/docs/guides/ai/vector-columns", + "title": "Vector columns" } ], - "resultChars": 22016 + "resultChars": 57701 } ] }, - "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", - "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", + "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", + "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-storage-001-private-bucket-access.json" + "sourcePath": "codex-gpt-5.6-no-skills/build-vectors-001-rag-with-permissions.json" }, { "experiment": "codex-gpt-5.6-no-skills", @@ -16691,33 +17781,30 @@ "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "build-tests-001-rls-tenant-isolation", - "stage": "build", + "eval": "deploy-database-001-prometheus-metrics", + "stage": "deploy", "product": [ "database" ], "topic": [ - "tests", - "rls" + "observability" ], "suite": "benchmark", - "interface": "cli", "passed": true, "checks": [ { - "name": "pgTAP test file(s) written under supabase/tests/", - "passed": true, - "notes": "1 file(s): supabase/tests/database/tenant_isolation.test.sql" + "name": "preserved existing app scrape job", + "passed": true }, { - "name": "pgTAP isolation tests ran and pass", + "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "notes": "5 passed, 2 failed" + "judgeNotes": "Meets requirements: preserves app scrape, adds HTTPS Supabase Metrics API target with correct path and project host, uses basic_auth with password_file, and docker-compose wires the matching secret to /run/secrets." }, { - "name": "agent correctly identifies the posts isolation bug from test results", + "name": "documented live deployment and verification steps", "passed": true, - "judgeNotes": "Correctly identifies `posts` as leaking cross-tenant rows for authenticated users, distinguishes `notes` as correctly isolated, and grounds the conclusion in failing pgTAP results." + "judgeNotes": "README includes Secret API key creation, matching Compose secret file path, stack recreation, and concrete verification via Prometheus target health." } ], "skills": { @@ -16728,239 +17815,323 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"database testing pgTAP row level security auth.uid tests\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Prometheus metrics endpoint hosted project customer privileged metrics authentication service role\", limit: 8) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } ... on ManagementApiReference { title href content } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/testing/overview", - "title": "Testing Overview" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic#4-alerts-and-automation", + "title": "4. Alerts and automation" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic#components", + "title": "Components" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic#1-define-the-scrape-job", + "title": "1. Define the scrape job" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic#collector-specific-notes", + "title": "Collector-specific notes" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic#2-secure-the-credentials", + "title": "2. Secure the credentials" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic#3-downstream-dashboards", + "title": "3. Downstream dashboards" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic#5-multi-project-setups", + "title": "5. Multi-project setups" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted#architecture", + "title": "Architecture" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted#1-deploy-prometheus", + "title": "1. Deploy Prometheus" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted#2-deploy-grafana", + "title": "2. Deploy Grafana" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted#3-import-supabase-dashboards", + "title": "3. Import Supabase dashboards" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted#4-configure-alerting", + "title": "4. Configure alerting" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted#5-operating-tips", + "title": "5. Operating tips" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#6-troubleshooting", + "title": "6. Troubleshooting" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#manual-setup", + "title": "Manual setup" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#prerequisites", + "title": "Prerequisites" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#1-create-a-grafana-cloud-stack", + "title": "1. Create a Grafana Cloud stack" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#2-install-the-supabase-integration-for-grafana-cloud", + "title": "2. Install the Supabase integration for Grafana Cloud" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#3-configure-the-supabase-integration", + "title": "3. Configure the Supabase integration" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#4-import-the-supabase-dashboard", + "title": "4. Import the Supabase dashboard" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#5-configure-alerts-optional", + "title": "5. Configure alerts (optional)" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud#installation", + "title": "Installation" + }, + { + "url": "https://supabase.com/docs/reference/api/v1-scrape-project-metrics", + "title": "Scrape a project's metrics" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", + "title": "Metrics API" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics#additional-resources", + "title": "Additional resources" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics#what-you-can-do-with-the-metrics-api", + "title": "What you can do with the Metrics API" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics#choose-your-monitoring-stack", + "title": "Choose your monitoring stack" + }, + { + "url": "https://supabase.com/docs/guides/security/security-testing", + "title": "Security testing of your Supabase projects" + }, + { + "url": "https://supabase.com/docs/guides/security/security-testing#permitted-services", + "title": "Permitted services" + }, + { + "url": "https://supabase.com/docs/guides/security/security-testing#prohibited-testing-and-activities", + "title": "Prohibited testing and activities" + }, + { + "url": "https://supabase.com/docs/guides/security/security-testing#terms-and-conditions", + "title": "Terms and conditions" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#before-you-begin", + "title": "Before you begin" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#enabling-the-envoy-gateway", + "title": "Enabling the Envoy gateway" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#verify", + "title": "Verify" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#architecture", + "title": "Architecture" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#configuration-file-structure", + "title": "Configuration file structure" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#how-the-configuration-is-rendered-at-startup", + "title": "How the configuration is rendered at startup" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#routes", + "title": "Routes" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#authentication", + "title": "Authentication" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#dashboard-basic-auth", + "title": "Dashboard basic auth" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#api-key-enforcement-on-protected-routes", + "title": "API key enforcement on protected routes" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#opaque-key-translation", + "title": "Opaque key translation" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#forwarded-headers-and-cors", + "title": "Forwarded headers and CORS" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#x-forwarded-headers", + "title": "X-Forwarded headers" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#cors", + "title": "CORS" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#security-hardening", + "title": "Security hardening" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#customizing-the-configuration", + "title": "Customizing the configuration" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#admin-interface", + "title": "Admin interface" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#troubleshooting", + "title": "Troubleshooting" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#logs", + "title": "Logs" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#common-issues", + "title": "Common issues" }, { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#see-also", + "title": "See also" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" + "url": "https://supabase.com/docs/guides/database/extensions/pgaudit", + "title": "PGAudit: Postgres Auditing" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pgtap", - "title": "pgTAP: Unit Testing" + "url": "https://supabase.com/docs/guides/database/extensions/pgaudit#enable-the-extension", + "title": "Enable the extension" }, { - "url": "https://supabase.com/docs/guides/database/testing", - "title": "Testing Your Database" - } - ], - "resultChars": 70562 - } - ] - }, - "prompt": "Can you audit the tenant isolation on our tables? Write some database tests covering both the happy path and the negative case, run them, and share the results.", - "promptSourcePath": "evals/build-tests-001-rls-tenant-isolation/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-tests-001-rls-tenant-isolation.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "build-vectors-001-rag-with-permissions", - "stage": "build", - "product": [ - "database", - "vectors" - ], - "topic": [ - "sql", - "rls" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "document_sections.embedding is vector(384)", - "passed": true, - "notes": "vector(384)" - }, - { - "name": "HNSW index on the embedding column", - "passed": true, - "notes": "CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops) WHERE (embedding IS NOT NULL)" - }, - { - "name": "index operator class matches the search operator", - "passed": true, - "notes": "function operators: <=>\nindexes: CREATE INDEX document_sections_embedding_hnsw_idx ON public.document_sections USING hnsw (embedding vector_cosine_ops) WHERE (embedding IS NOT NULL)" - }, - { - "name": "user A search returns only own sections, best match first", - "passed": true - }, - { - "name": "user B search returns only own sections, best match first", - "passed": true - }, - { - "name": "user A reads only own sections through the API", - "passed": true - }, - { - "name": "user A reads only own documents through the API", - "passed": true - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase AI gte-small embedding dimensions pgvector semantic search match_documents RLS security invoker\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/database/extensions/pgaudit#configure-the-extension", + "title": "Configure the extension" + }, { - "url": "https://supabase.com/docs/guides/ai", - "title": "AI & Vectors" + "url": "https://supabase.com/docs/guides/database/extensions/pgaudit#session-mode-categories", + "title": "Session mode categories" }, { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" + "url": "https://supabase.com/docs/guides/database/extensions/pgaudit#session-logging", + "title": "Session logging" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pgvector", - "title": "pgvector: Embeddings and vector similarity" + "url": "https://supabase.com/docs/guides/database/extensions/pgaudit#user-logging", + "title": "User logging" }, { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" + "url": "https://supabase.com/docs/guides/database/extensions/pgaudit#global-logging", + "title": "Global logging" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - } - ], - "resultChars": 63278 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"gte-small 384 dimensions Supabase.ai.Session\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/database/extensions/pgaudit#object-logging", + "title": "Object logging" + }, { - "url": "https://supabase.com/docs/guides/ai/choosing-compute-addon", - "title": "Choosing your Compute Add-on" + "url": "https://supabase.com/docs/guides/database/extensions/pgaudit#interpreting-audit-logs", + "title": "Interpreting Audit Logs" }, { - "url": "https://supabase.com/docs/guides/functions/ai-models", - "title": "Running AI Models" + "url": "https://supabase.com/docs/guides/database/extensions/pgaudit#finding-and-filtering-audit-logs", + "title": "Finding and filtering audit logs" }, { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" + "url": "https://supabase.com/docs/guides/database/extensions/pgaudit#practical-examples", + "title": "Practical examples" }, { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" + "url": "https://supabase.com/docs/guides/database/extensions/pgaudit#monitoring-api-events", + "title": "Monitoring API events" }, { - "url": "https://supabase.com/docs/guides/storage/vector/working-with-indexes", - "title": "Working with Vector Indexes" - } - ], - "resultChars": 76269 - } - ] - }, - "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", - "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/build-vectors-001-rag-with-permissions.json" - }, - { - "experiment": "codex-gpt-5.6-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" - }, - "eval": "deploy-database-001-prometheus-metrics", - "stage": "deploy", - "product": [ - "database" - ], - "topic": [ - "observability" - ], - "suite": "benchmark", - "passed": true, - "checks": [ - { - "name": "preserved existing app scrape job", - "passed": true - }, - { - "name": "configured the Supabase Metrics API scrape correctly", - "passed": true, - "judgeNotes": "Meets all requirements: app scrape preserved, Supabase HTTPS metrics endpoint configured with Basic Auth password_file, project target present, and Docker Compose wires the secret to /run/secrets." - }, - { - "name": "documented live deployment and verification steps", - "passed": true, - "judgeNotes": "README includes creating a Supabase secret API key, writing it to the matching Docker secret file, recreating the Compose Prometheus service, and verifying via Prometheus targets or an up{job=\"supabase\"} query." - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Prometheus project metrics endpoint customer v1 privileged metrics service_role basic auth observability\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/database/extensions/pgaudit#monitoring-the-authusers-table", + "title": "Monitoring the auth.users table" + }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" + "url": "https://supabase.com/docs/guides/database/extensions/pgaudit#best-practices", + "title": "Best practices" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" + "url": "https://supabase.com/docs/guides/database/extensions/pgaudit#disabling-excess-logging", + "title": "Disabling excess logging" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" + "url": "https://supabase.com/docs/guides/database/extensions/pgaudit#faq", + "title": "FAQ" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" + "url": "https://supabase.com/docs/guides/database/extensions/pgaudit#using-pgaudit-to-debug-database-functions", + "title": "Using PGAudit to debug database functions" }, { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" + "url": "https://supabase.com/docs/guides/database/extensions/pgaudit#downloading-database-logs", + "title": "Downloading database logs" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/reports", - "title": "Reports" + "url": "https://supabase.com/docs/guides/database/extensions/pgaudit#logging-observed-table-rows", + "title": "Logging observed table rows" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pgaudit", - "title": "PGAudit: Postgres Auditing" + "url": "https://supabase.com/docs/guides/database/extensions/pgaudit#logging-function-parameters", + "title": "Logging function parameters" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pgaudit#does-pgaudit-support-system-wide-configurations", + "title": "Does PGAudit support system wide configurations?" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pgaudit#resources", + "title": "Resources" } ], - "resultChars": 82042 + "resultChars": 185838 } ] }, @@ -17017,7 +18188,7 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Edge Functions secrets environment variables Deno.env deploy functions invoke no verify jwt\", limit: 5) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } } } }", + "query": "query { searchDocs(query: \"Supabase Edge Functions environment variables secrets set deploy function verify JWT Deno serve CORS\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -17025,19 +18196,27 @@ "title": "Environment Variables" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", + "title": "Inspecting edge function environment variables" }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", "title": "Self-Hosted Functions" }, { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", + "title": "Serverless Drivers" } ], - "resultChars": 37288 + "resultChars": 50291 } ] }, @@ -17094,7 +18273,7 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"self-hosting Docker compose install docker .env secrets JWT_SECRET ANON_KEY SERVICE_ROLE_KEY POSTGRES_PASSWORD SECRET_KEY_BASE VAULT_ENC_KEY\", limit: 8) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"self-hosting Docker docker-compose .env secrets JWT keys 2026\", limit: 8) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } ... on CLICommandReference { title href content } } } }", "hasContent": true, "pages": [ { @@ -17102,20 +18281,16 @@ "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#environment-variables-configuration", - "title": "Environment variables configuration" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#verifying-the-setup", - "title": "Verifying the setup" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#request-flows", + "title": "Request flows" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#new-api-keys-format", - "title": "New API keys format" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#unauthenticated-requests-api-key-only-no-user-session-jwt", + "title": "Unauthenticated requests (API key only, no user session JWT)" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#adding-the-new-keys", - "title": "Adding the new keys" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#authenticated-requests-user-session-jwt", + "title": "Authenticated requests (user session JWT)" }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#additional-resources", @@ -17125,6 +18300,22 @@ "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#before-you-begin", "title": "Before you begin" }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#adding-the-new-keys", + "title": "Adding the new keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#new-api-keys-format", + "title": "New API keys format" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#verifying-the-setup", + "title": "Verifying the setup" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#environment-variables-configuration", + "title": "Environment variables configuration" + }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#differences-from-the-supabase-platform", "title": "Differences from the Supabase platform" @@ -17154,620 +18345,608 @@ "title": "Kong API gateway routing" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#request-flows", - "title": "Request flows" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#unauthenticated-requests-api-key-only-no-user-session-jwt", - "title": "Unauthenticated requests (API key only, no user session JWT)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#authenticated-requests-user-session-jwt", - "title": "Authenticated requests (user session JWT)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#before-you-begin", - "title": "Before you begin" + "url": "https://supabase.com/docs/guides/self-hosting/docker#exposing-your-postgres-database", + "title": "Exposing your Postgres database" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#enabling-the-envoy-gateway", - "title": "Enabling the Envoy gateway" + "url": "https://supabase.com/docs/guides/self-hosting/docker#contents", + "title": "Contents" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#verify", - "title": "Verify" + "url": "https://supabase.com/docs/guides/self-hosting/docker#before-you-begin", + "title": "Before you begin" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#architecture", - "title": "Architecture" + "url": "https://supabase.com/docs/guides/self-hosting/docker#system-requirements", + "title": "System requirements" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#configuration-file-structure", - "title": "Configuration file structure" + "url": "https://supabase.com/docs/guides/self-hosting/docker#installing-supabase", + "title": "Installing Supabase" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#how-the-configuration-is-rendered-at-startup", - "title": "How the configuration is rendered at startup" + "url": "https://supabase.com/docs/guides/self-hosting/docker#quick-start-linux", + "title": "Quick start (Linux)" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#routes", - "title": "Routes" + "url": "https://supabase.com/docs/guides/self-hosting/docker#manual-installation", + "title": "Manual installation" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#authentication", - "title": "Authentication" + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-and-securing-supabase", + "title": "Configuring and securing Supabase" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#dashboard-basic-auth", - "title": "Dashboard basic auth" + "url": "https://supabase.com/docs/guides/self-hosting/docker#generate-keys-and-secrets", + "title": "Generate keys and secrets" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#api-key-enforcement-on-protected-routes", - "title": "API key enforcement on protected routes" + "url": "https://supabase.com/docs/guides/self-hosting/docker#configure-supabase-urls", + "title": "Configure Supabase URLs" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#opaque-key-translation", - "title": "Opaque key translation" + "url": "https://supabase.com/docs/guides/self-hosting/docker#where-to-find-your-credentials", + "title": "Where to find your credentials" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#forwarded-headers-and-cors", - "title": "Forwarded headers and CORS" + "url": "https://supabase.com/docs/guides/self-hosting/docker#studio-authentication", + "title": "Studio authentication" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#x-forwarded-headers", - "title": "X-Forwarded headers" + "url": "https://supabase.com/docs/guides/self-hosting/docker#starting-and-stopping", + "title": "Starting and stopping" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#cors", - "title": "CORS" + "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-supabase-studio-dashboard", + "title": "Accessing Supabase Studio (Dashboard)" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#security-hardening", - "title": "Security hardening" + "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-postgres", + "title": "Accessing Postgres" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#customizing-the-configuration", - "title": "Customizing the configuration" + "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-edge-functions", + "title": "Accessing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#admin-interface", - "title": "Admin interface" + "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-apis", + "title": "Accessing APIs" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#troubleshooting", - "title": "Troubleshooting" + "url": "https://supabase.com/docs/guides/self-hosting/docker#enabling-analytics", + "title": "Enabling analytics" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#logs", - "title": "Logs" + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-https", + "title": "Configuring HTTPS" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#common-issues", - "title": "Common issues" + "url": "https://supabase.com/docs/guides/self-hosting/docker#managing-the-stack", + "title": "Managing the stack" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#see-also", - "title": "See also" + "url": "https://supabase.com/docs/guides/self-hosting/docker#updating", + "title": "Updating" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", - "title": "Configure Social Login (OAuth) Providers" + "url": "https://supabase.com/docs/guides/self-hosting/docker#uninstalling", + "title": "Uninstalling" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#additional-resources", - "title": "Additional resources" + "url": "https://supabase.com/docs/guides/self-hosting/docker#advanced-topics", + "title": "Advanced topics" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-5-verify-the-configuration", - "title": "Step 5: Verify the configuration" + "url": "https://supabase.com/docs/guides/self-hosting/docker#architecture", + "title": "Architecture" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-4-restart-the-auth-service", - "title": "Step 4: Restart the auth service" + "url": "https://supabase.com/docs/guides/self-hosting/docker#setting-database-password", + "title": "Setting database password" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#test-the-login-flow", - "title": "Test the login flow" + "url": "https://supabase.com/docs/guides/self-hosting/docker#changing-database-password", + "title": "Changing database password" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#troubleshooting", - "title": "Troubleshooting" + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-secrets", + "title": "Configuring secrets" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-3-enable-the-matching-lines-in-docker-compose-configuration", - "title": "Step 3: Enable the matching lines in Docker Compose configuration" + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-supabase-services", + "title": "Configuring Supabase services" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-2-configure-environment-variables", - "title": "Step 2: Configure environment variables" + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-social-login-oauth-providers", + "title": "Configuring social login (OAuth) providers" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-1-register-your-app-with-the-provider", - "title": "Step 1: Register your app with the provider" + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-phone-login-sms-and-mfa", + "title": "Configuring phone login, SMS, and MFA" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-by-step-configuration", - "title": "Step-by-step configuration" + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-an-email-server", + "title": "Configuring an email server" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#auth-environment-variables", - "title": "Auth environment variables" + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-s3-storage", + "title": "Configuring S3 Storage" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#oauth-request-flow", - "title": "OAuth request flow" + "url": "https://supabase.com/docs/guides/self-hosting/docker#using-file-backend-in-storage-on-macos", + "title": "Using file backend in Storage on macOS" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#before-you-begin", - "title": "Before you begin" + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-supabase-ai-assistant", + "title": "Configuring Supabase AI Assistant" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#site-url-or-redirect-url-errors-after-login", - "title": "Site URL or redirect URL errors after login" + "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-postgres-through-supavisor", + "title": "Accessing Postgres through Supavisor" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#nonce-check-failure-on-mobile-google-sign-in", - "title": "Nonce check failure on mobile (Google Sign In)" + "url": "https://supabase.com/docs/guides/self-hosting/docker#setting-log_min_messages-in-postgres", + "title": "Setting log_min_messages in Postgres" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#auth-service-fails-to-start", - "title": "Auth service fails to start" + "url": "https://supabase.com/docs/guides/self-hosting/docker#managing-your-secrets", + "title": "Managing your secrets" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#environment-variable-reference", - "title": "Environment variable reference" + "url": "https://supabase.com/docs/guides/self-hosting/docker#demo", + "title": "Demo" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#other-supported-providers", - "title": "Other supported providers" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", + "title": "Configure Social Login (OAuth) Providers" }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#variables-added-to-the-environment-but-provider-still-not-working", "title": "Variables added to the environment but provider still not working" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#provider-specific-setup", - "title": "Provider-specific setup" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#site-url-or-redirect-url-errors-after-login", + "title": "Site URL or redirect URL errors after login" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#generate-keys-and-secrets", - "title": "Generate keys and secrets" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#nonce-check-failure-on-mobile-google-sign-in", + "title": "Nonce check failure on mobile (Google Sign In)" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-and-securing-supabase", - "title": "Configuring and securing Supabase" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#auth-service-fails-to-start", + "title": "Auth service fails to start" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#manual-installation", - "title": "Manual installation" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#test-the-login-flow", + "title": "Test the login flow" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#quick-start-linux", - "title": "Quick start (Linux)" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#troubleshooting", + "title": "Troubleshooting" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#installing-supabase", - "title": "Installing Supabase" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#environment-variable-reference", + "title": "Environment variable reference" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#system-requirements", - "title": "System requirements" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#additional-resources", + "title": "Additional resources" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#before-you-begin", - "title": "Before you begin" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#provider-specific-setup", + "title": "Provider-specific setup" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#contents", - "title": "Contents" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#other-supported-providers", + "title": "Other supported providers" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#exposing-your-postgres-database", - "title": "Exposing your Postgres database" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#before-you-begin", + "title": "Before you begin" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#setting-log_min_messages-in-postgres", - "title": "Setting log_min_messages in Postgres" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#oauth-request-flow", + "title": "OAuth request flow" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#using-file-backend-in-storage-on-macos", - "title": "Using file backend in Storage on macOS" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#auth-environment-variables", + "title": "Auth environment variables" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-supabase-ai-assistant", - "title": "Configuring Supabase AI Assistant" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-by-step-configuration", + "title": "Step-by-step configuration" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-postgres-through-supavisor", - "title": "Accessing Postgres through Supavisor" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-1-register-your-app-with-the-provider", + "title": "Step 1: Register your app with the provider" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-s3-storage", - "title": "Configuring S3 Storage" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-2-configure-environment-variables", + "title": "Step 2: Configure environment variables" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-an-email-server", - "title": "Configuring an email server" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-3-enable-the-matching-lines-in-docker-compose-configuration", + "title": "Step 3: Enable the matching lines in Docker Compose configuration" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-phone-login-sms-and-mfa", - "title": "Configuring phone login, SMS, and MFA" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-4-restart-the-auth-service", + "title": "Step 4: Restart the auth service" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-social-login-oauth-providers", - "title": "Configuring social login (OAuth) providers" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-5-verify-the-configuration", + "title": "Step 5: Verify the configuration" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-supabase-services", - "title": "Configuring Supabase services" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-secrets", - "title": "Configuring secrets" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#configuration-file-structure", + "title": "Configuration file structure" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#changing-database-password", - "title": "Changing database password" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#enabling-the-envoy-gateway", + "title": "Enabling the Envoy gateway" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#setting-database-password", - "title": "Setting database password" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#before-you-begin", + "title": "Before you begin" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#architecture", + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#architecture", "title": "Architecture" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#advanced-topics", - "title": "Advanced topics" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#verify", + "title": "Verify" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#uninstalling", - "title": "Uninstalling" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#see-also", + "title": "See also" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#updating", - "title": "Updating" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#common-issues", + "title": "Common issues" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#logs", + "title": "Logs" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#managing-the-stack", - "title": "Managing the stack" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#troubleshooting", + "title": "Troubleshooting" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-https", - "title": "Configuring HTTPS" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#admin-interface", + "title": "Admin interface" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#enabling-analytics", - "title": "Enabling analytics" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#customizing-the-configuration", + "title": "Customizing the configuration" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-apis", - "title": "Accessing APIs" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#security-hardening", + "title": "Security hardening" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-edge-functions", - "title": "Accessing Edge Functions" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#cors", + "title": "CORS" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-postgres", - "title": "Accessing Postgres" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#x-forwarded-headers", + "title": "X-Forwarded headers" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-supabase-studio-dashboard", - "title": "Accessing Supabase Studio (Dashboard)" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#forwarded-headers-and-cors", + "title": "Forwarded headers and CORS" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#demo", - "title": "Demo" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#opaque-key-translation", + "title": "Opaque key translation" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#managing-your-secrets", - "title": "Managing your secrets" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#api-key-enforcement-on-protected-routes", + "title": "API key enforcement on protected routes" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#starting-and-stopping", - "title": "Starting and stopping" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#dashboard-basic-auth", + "title": "Dashboard basic auth" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#studio-authentication", - "title": "Studio authentication" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#authentication", + "title": "Authentication" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#where-to-find-your-credentials", - "title": "Where to find your credentials" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#routes", + "title": "Routes" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configure-supabase-urls", - "title": "Configure Supabase URLs" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#how-the-configuration-is-rendered-at-startup", + "title": "How the configuration is rendered at startup" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#memory-or-timeout-errors", - "title": "Memory or timeout errors" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#create-the-success-screen", + "title": "Create the success screen" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#custom-env-vars-not-available-in-functions", - "title": "Custom env vars not available in functions" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#implement-the-mainactivity", + "title": "Implement the MainActivity" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#changes-to-function-code-not-reflected-after-editing", - "title": "Changes to function code not reflected after editing" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#implement-screens", + "title": "Implement screens" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#500-error-on-invocation", - "title": "500 error on invocation" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#implement-repositories", + "title": "Implement repositories" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#troubleshooting", - "title": "Troubleshooting" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#create-a-data-transfer-object", + "title": "Create a data transfer object" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#copying-functions-from-supabase-platform", - "title": "Copying functions from Supabase platform" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#provide-supabase-instances-with-hilt", + "title": "Provide Supabase instances with Hilt" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#deploying-functions-to-a-remote-server", - "title": "Deploying functions to a remote server" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#set-up-hilt-for-dependency-injection", + "title": "Set up Hilt for dependency injection" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#managing-functions-via-dashboard", - "title": "Managing functions via dashboard" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#set-up-supabase-dependencies", + "title": "Set up Supabase dependencies" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#internal-vs-external-urls", - "title": "Internal vs external URLs" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#use-value-from-buildconfig", + "title": "Use value from BuildConfig" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#calling-supabase-services-from-functions", - "title": "Calling Supabase services from functions" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#read-and-set-value-to-buildconfig", + "title": "Read and set value to BuildConfig" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#accessing-variables-in-functions", - "title": "Accessing variables in functions" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#create-local-environment-secret", + "title": "Create local environment secret" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#using-inline-environment-variables", - "title": "Using inline environment variables" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#set-up-api-key-and-secret-securely", + "title": "Set up API key and secret securely" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#using-an-env-file-recommended", - "title": "Using an env file (recommended)" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#create-new-android-project", + "title": "Create new Android project" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#custom-environment-variables", - "title": "Custom environment variables" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#building-the-app", + "title": "Building the app" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#step-3-invoke-your-function", - "title": "Step 3: Invoke your function" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#set-up-google-authentication", + "title": "Set up Google authentication" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#step-2-restart-the-functions-service-to-pick-up-the-new-function", - "title": "Step 2: Restart the functions service to pick up the new function" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#get-api-details", + "title": "Get API details" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#step-1-add-a-new-function-directory-and-the-function-code", - "title": "Step 1: Add a new function directory and the function code" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#set-up-the-database-schema", + "title": "Set up the database schema" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#create-a-new-function", - "title": "Create a new function" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#create-a-project", + "title": "Create a project" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#invoke-the-default-function", - "title": "Invoke the default function" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin#project-setup", + "title": "Project setup" }, { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates", - "title": "Custom Email Templates" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#step-1-create-a-templates-directory", - "title": "Step 1: Create a templates directory" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#invoke-the-default-function", + "title": "Invoke the default function" }, { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#overview", - "title": "Overview" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#create-a-new-function", + "title": "Create a new function" }, { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#authentication-email-templates", - "title": "Authentication email templates" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#step-1-add-a-new-function-directory-and-the-function-code", + "title": "Step 1: Add a new function directory and the function code" }, { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#example", - "title": "Example" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#step-2-restart-the-functions-service-to-pick-up-the-new-function", + "title": "Step 2: Restart the functions service to pick up the new function" }, { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#step-2-update-docker-composeyml", - "title": "Step 2: Update docker-compose.yml" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#step-3-invoke-your-function", + "title": "Step 3: Invoke your function" }, { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#what-this-configuration-does", - "title": "What this configuration does" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#custom-environment-variables", + "title": "Custom environment variables" }, { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#step-3-restart-containers", - "title": "Step 3: Restart containers" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#using-an-env-file-recommended", + "title": "Using an env file (recommended)" }, { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#notification-email-templates", - "title": "Notification email templates" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#using-inline-environment-variables", + "title": "Using inline environment variables" }, { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#example-1", - "title": "Example" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#accessing-variables-in-functions", + "title": "Accessing variables in functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#step-1-create-the-templates-directory", - "title": "Step 1: Create the templates directory" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#calling-supabase-services-from-functions", + "title": "Calling Supabase services from functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#step-2-update-docker-composeyml-1", - "title": "Step 2: Update docker-compose.yml" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#internal-vs-external-urls", + "title": "Internal vs external URLs" }, { - "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#step-3-restart-containers-1", - "title": "Step 3: Restart containers" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#managing-functions-via-dashboard", + "title": "Managing functions via dashboard" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", - "title": "Configure S3 Storage" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#deploying-functions-to-a-remote-server", + "title": "Deploying functions to a remote server" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#signature-mismatch-errors", - "title": "Signature mismatch errors" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#copying-functions-from-supabase-platform", + "title": "Copying functions from Supabase platform" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#test-with-the-aws-cli", - "title": "Test with the AWS CLI" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#troubleshooting", + "title": "Troubleshooting" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#test-with-rclone", - "title": "Test with rclone" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#500-error-on-invocation", + "title": "500 error on invocation" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#how-to-configure-an-s3-backend", - "title": "How to configure an S3 backend" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#changes-to-function-code-not-reflected-after-editing", + "title": "Changes to function code not reflected after editing" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#using-rustfs", - "title": "Using RustFS" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#custom-env-vars-not-available-in-functions", + "title": "Custom env vars not available in functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#using-minio", - "title": "Using MinIO" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#memory-or-timeout-errors", + "title": "Memory or timeout errors" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#using-aws-s3", - "title": "Using AWS S3" + "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates", + "title": "Custom Email Templates" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#s3-compatible-providers", - "title": "S3-compatible providers" + "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#step-3-restart-containers-1", + "title": "Step 3: Restart containers" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#verify", - "title": "Verify" + "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#step-2-update-docker-composeyml-1", + "title": "Step 2: Update docker-compose.yml" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#session-token", - "title": "Session token" + "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#step-1-create-the-templates-directory", + "title": "Step 1: Create the templates directory" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#troubleshooting", - "title": "Troubleshooting" + "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#overview", + "title": "Overview" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#tus-upload-errors-on-cloudflare-r2", - "title": "TUS upload errors on Cloudflare R2" + "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#authentication-email-templates", + "title": "Authentication email templates" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#permission-denied-on-uploads", - "title": "Permission denied on uploads" + "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#example", + "title": "Example" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#upload-urls-point-to-localhost", - "title": "Upload URLs point to localhost" + "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#step-1-create-a-templates-directory", + "title": "Step 1: Create a templates directory" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#additional-resources", - "title": "Additional resources" + "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#step-2-update-docker-composeyml", + "title": "Step 2: Update docker-compose.yml" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3#enable-the-s3-protocol-endpoint", - "title": "Enable the S3 protocol endpoint" + "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#what-this-configuration-does", + "title": "What this configuration does" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17", - "title": "Upgrade to Postgres 17" + "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#step-3-restart-containers", + "title": "Step 3: Restart containers" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#new-deployment-with-postgres-17", - "title": "New deployment with Postgres 17" + "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#notification-email-templates", + "title": "Notification email templates" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#before-you-begin", - "title": "Before you begin" + "url": "https://supabase.com/docs/guides/self-hosting/custom-email-templates#example-1", + "title": "Example" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#run-the-upgrade", - "title": "Run the upgrade" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#restoring-from-a-manual-backup", - "title": "Restoring from a manual backup" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#additional-resources", + "title": "Additional resources" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#postgres-17-fails-to-start-with-a-leftover-db-config-volume", - "title": "Postgres 17 fails to start with a leftover db-config volume" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#step-5-verify-the-restore", + "title": "Step 5: Verify the restore" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#disk-space-issues-during-upgrade", - "title": "Disk space issues during upgrade" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#step-4-restore-to-your-self-hosted-database", + "title": "Step 4: Restore to your self-hosted database" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#services-fail-to-connect-after-upgrade", - "title": "Services fail to connect after upgrade" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#before-you-begin", + "title": "Before you begin" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#pgsodium--supabase-vault-errors", - "title": "pgsodium / Supabase Vault errors" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#step-1-get-your-platform-connection-string", + "title": "Step 1: Get your platform connection string" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#pg_upgrade-fails-with-replication-slot-errors", - "title": "pg_upgrade fails with replication slot errors" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#step-2-back-up-your-platform-database", + "title": "Step 2: Back up your platform database" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#troubleshooting", - "title": "Troubleshooting" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#step-3-prepare-your-self-hosted-instance", + "title": "Step 3: Prepare your self-hosted instance" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#upgrade-process-details", - "title": "Upgrade process details" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#whats-included-in-the-restore-and-whats-not", + "title": "What's included in the restore and what's not" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#custom-postgres-configuration", - "title": "Custom Postgres configuration" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#auth-considerations", + "title": "Auth considerations" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#rollback", - "title": "Rollback" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#postgres-version-compatibility", + "title": "Postgres version compatibility" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#after-the-upgrade", - "title": "After the upgrade" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#troubleshooting", + "title": "Troubleshooting" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#extensions-removed-in-postgres-17", - "title": "Extensions removed in Postgres 17" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#version-mismatches-between-platform-and-self-hosted", + "title": "Version mismatches between platform and self-hosted" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#requirements", - "title": "Requirements" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#extension-not-available", + "title": "Extension not available" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#create-a-backup", - "title": "Create a backup" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#connection-refused", + "title": "Connection refused" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#what-the-upgrade-does", - "title": "What the upgrade does" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#legacy-studio-configuration", + "title": "Legacy Studio configuration" }, { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#upgrade-an-existing-postgres-15-deployment", - "title": "Upgrade an existing Postgres 15 deployment" + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform#custom-roles-missing-passwords", + "title": "Custom roles missing passwords" }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#provider-not-enabled-or-provider-seen-as-false-in-settings" @@ -17777,77 +18956,126 @@ }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#400-missing-function-name-in-request" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17#permission-denied-on-the-data-directory" } ], - "resultChars": 450330 + "resultChars": 554611 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Docker self-hosting quick start Linux git clone supabase docker copy .env.example generate-keys.sh\", limit: 5) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", + "query": "query { searchDocs(query: \"Self-hosting with Docker Quick Start Linux generate keys secrets .env docker compose\", limit: 5) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-supabase-ai-assistant", - "title": "Configuring Supabase AI Assistant" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#new-api-keys-format", + "title": "New API keys format" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#changing-database-password", - "title": "Changing database password" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#backward-compatibility", + "title": "Backward compatibility" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-secrets", - "title": "Configuring secrets" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#differences-from-the-supabase-platform", + "title": "Differences from the Supabase platform" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-supabase-services", - "title": "Configuring Supabase services" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#environment-variables-configuration", + "title": "Environment variables configuration" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-social-login-oauth-providers", - "title": "Configuring social login (OAuth) providers" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#verifying-the-setup", + "title": "Verifying the setup" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#adding-the-new-keys", + "title": "Adding the new keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#before-you-begin", + "title": "Before you begin" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#additional-resources", + "title": "Additional resources" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#authenticated-requests-user-session-jwt", + "title": "Authenticated requests (user session JWT)" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#unauthenticated-requests-api-key-only-no-user-session-jwt", + "title": "Unauthenticated requests (API key only, no user session JWT)" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#request-flows", + "title": "Request flows" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#kong-api-gateway-routing", + "title": "Kong API gateway routing" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#what-client-sdk-sends", + "title": "What client SDK sends" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#how-it-works", + "title": "How it works" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#regenerating-asymmetric-key-pair", + "title": "Regenerating asymmetric key pair" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#rotating-the-new-api-keys", + "title": "Rotating the new API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" }, { "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-phone-login-sms-and-mfa", "title": "Configuring phone login, SMS, and MFA" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-an-email-server", - "title": "Configuring an email server" + "url": "https://supabase.com/docs/guides/self-hosting/docker#demo", + "title": "Demo" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-s3-storage", - "title": "Configuring S3 Storage" + "url": "https://supabase.com/docs/guides/self-hosting/docker#managing-your-secrets", + "title": "Managing your secrets" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#using-file-backend-in-storage-on-macos", - "title": "Using file backend in Storage on macOS" + "url": "https://supabase.com/docs/guides/self-hosting/docker#setting-log_min_messages-in-postgres", + "title": "Setting log_min_messages in Postgres" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/docker#exposing-your-postgres-database", + "title": "Exposing your Postgres database" }, { "url": "https://supabase.com/docs/guides/self-hosting/docker#accessing-postgres-through-supavisor", "title": "Accessing Postgres through Supavisor" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#exposing-your-postgres-database", - "title": "Exposing your Postgres database" + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-supabase-ai-assistant", + "title": "Configuring Supabase AI Assistant" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#setting-log_min_messages-in-postgres", - "title": "Setting log_min_messages in Postgres" + "url": "https://supabase.com/docs/guides/self-hosting/docker#using-file-backend-in-storage-on-macos", + "title": "Using file backend in Storage on macOS" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#managing-your-secrets", - "title": "Managing your secrets" + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-s3-storage", + "title": "Configuring S3 Storage" }, { - "url": "https://supabase.com/docs/guides/self-hosting/docker#demo", - "title": "Demo" + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-an-email-server", + "title": "Configuring an email server" }, { "url": "https://supabase.com/docs/guides/self-hosting/docker#contents", @@ -17946,275 +19174,280 @@ "title": "Setting database password" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/guides/self-hosting/docker#changing-database-password", + "title": "Changing database password" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#how-it-works", - "title": "How it works" + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-secrets", + "title": "Configuring secrets" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#before-you-begin", - "title": "Before you begin" + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-supabase-services", + "title": "Configuring Supabase services" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#adding-the-new-keys", - "title": "Adding the new keys" + "url": "https://supabase.com/docs/guides/self-hosting/docker#configuring-social-login-oauth-providers", + "title": "Configuring social login (OAuth) providers" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#new-api-keys-format", - "title": "New API keys format" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth", + "title": "Configure Social Login (OAuth) Providers" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#verifying-the-setup", - "title": "Verifying the setup" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-2-configure-environment-variables", + "title": "Step 2: Configure environment variables" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#environment-variables-configuration", - "title": "Environment variables configuration" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-1-register-your-app-with-the-provider", + "title": "Step 1: Register your app with the provider" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#differences-from-the-supabase-platform", - "title": "Differences from the Supabase platform" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-by-step-configuration", + "title": "Step-by-step configuration" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#backward-compatibility", - "title": "Backward compatibility" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#auth-environment-variables", + "title": "Auth environment variables" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#rotating-the-new-api-keys", - "title": "Rotating the new API keys" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#oauth-request-flow", + "title": "OAuth request flow" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#regenerating-asymmetric-key-pair", - "title": "Regenerating asymmetric key pair" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#before-you-begin", + "title": "Before you begin" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#what-client-sdk-sends", - "title": "What client SDK sends" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#test-the-login-flow", + "title": "Test the login flow" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#kong-api-gateway-routing", - "title": "Kong API gateway routing" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#troubleshooting", + "title": "Troubleshooting" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#request-flows", - "title": "Request flows" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#additional-resources", + "title": "Additional resources" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#unauthenticated-requests-api-key-only-no-user-session-jwt", - "title": "Unauthenticated requests (API key only, no user session JWT)" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#environment-variable-reference", + "title": "Environment variable reference" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#authenticated-requests-user-session-jwt", - "title": "Authenticated requests (user session JWT)" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#auth-service-fails-to-start", + "title": "Auth service fails to start" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys#additional-resources", - "title": "Additional resources" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#nonce-check-failure-on-mobile-google-sign-in", + "title": "Nonce check failure on mobile (Google Sign In)" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#site-url-or-redirect-url-errors-after-login", + "title": "Site URL or redirect URL errors after login" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-4-add-seed-data", - "title": "Step 4: Add seed data" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#variables-added-to-the-environment-but-provider-still-not-working", + "title": "Variables added to the environment but provider still not working" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-5-verify", - "title": "Step 5: Verify" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#other-supported-providers", + "title": "Other supported providers" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-6-commit", - "title": "Step 6: Commit" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#provider-specific-setup", + "title": "Provider-specific setup" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#the-daily-workflow", - "title": "The daily workflow" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-5-verify-the-configuration", + "title": "Step 5: Verify the configuration" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#making-schema-changes", - "title": "Making schema changes" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-4-restart-the-auth-service", + "title": "Step 4: Restart the auth service" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#generating-types", - "title": "Generating types" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#step-3-enable-the-matching-lines-in-docker-compose-configuration", + "title": "Step 3: Enable the matching lines in Docker Compose configuration" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#staying-in-sync-with-your-team", - "title": "Staying in sync with your team" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#pushing-to-a-remote-project", - "title": "Pushing to a remote project" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#verify", + "title": "Verify" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#resetting-a-remote-dev-or-staging-project", - "title": "Resetting a remote dev or staging project" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#enabling-the-envoy-gateway", + "title": "Enabling the Envoy gateway" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#key-commands-at-a-glance", - "title": "Key commands at a glance" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#before-you-begin", + "title": "Before you begin" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#cleaning-up-generated-migrations", - "title": "Cleaning up generated migrations" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#opaque-key-translation", + "title": "Opaque key translation" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#grants", - "title": "Grants" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#forwarded-headers-and-cors", + "title": "Forwarded headers and CORS" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#revokere-grant-patterns", - "title": "Revoke/re-grant patterns" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#x-forwarded-headers", + "title": "X-Forwarded headers" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#api-key-enforcement-on-protected-routes", + "title": "API key enforcement on protected routes" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#cors", + "title": "CORS" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#security-hardening", + "title": "Security hardening" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#extension-statements", - "title": "Extension statements" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#customizing-the-configuration", + "title": "Customizing the configuration" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#known-limitations-of-db-diff", - "title": "Known limitations of db diff" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#admin-interface", + "title": "Admin interface" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#troubleshooting", + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#troubleshooting", "title": "Troubleshooting" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-3-create-your-schema", - "title": "Step 3: Create your schema" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-2-start-the-local-stack", - "title": "Step 2: Start the local stack" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#logs", + "title": "Logs" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-1-initialize-1", - "title": "Step 1: Initialize" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#common-issues", + "title": "Common issues" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#start-a-new-project-from-scratch", - "title": "Start a new project from scratch" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#see-also", + "title": "See also" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-7-commit", - "title": "Step 7: Commit" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#dashboard-basic-auth", + "title": "Dashboard basic auth" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-6-verify", - "title": "Step 6: Verify" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#authentication", + "title": "Authentication" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-5-create-seed-data", - "title": "Step 5: Create seed data" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#routes", + "title": "Routes" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-4-pull-the-remote-schema", - "title": "Step 4: Pull the remote schema" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#how-the-configuration-is-rendered-at-startup", + "title": "How the configuration is rendered at startup" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-3-link-to-your-remote-project", - "title": "Step 3: Link to your remote project" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#configuration-file-structure", + "title": "Configuration file structure" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-2-authenticate", - "title": "Step 2: Authenticate" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy#architecture", + "title": "Architecture" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#before-you-begin", - "title": "Before you begin" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#the-supabase-directory", - "title": "The ./supabase directory" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#deploying-functions-to-a-remote-server", + "title": "Deploying functions to a remote server" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#move-an-existing-project-to-local-development", - "title": "Move an existing project to local development" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#invoke-the-default-function", + "title": "Invoke the default function" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-1-initialize", - "title": "Step 1: Initialize" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#create-a-new-function", + "title": "Create a new function" }, { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", - "title": "Supabase CLI" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#step-1-add-a-new-function-directory-and-the-function-code", + "title": "Step 1: Add a new function directory and the function code" }, { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#how-to-opt-out", - "title": "How to opt out" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#step-2-restart-the-functions-service-to-pick-up-the-new-function", + "title": "Step 2: Restart the functions service to pick up the new function" }, { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#telemetry", - "title": "Telemetry" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#step-3-invoke-your-function", + "title": "Step 3: Invoke your function" }, { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#stopping-local-services", - "title": "Stopping local services" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#custom-environment-variables", + "title": "Custom environment variables" }, { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#access-your-projects-services", - "title": "Access your project's services" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#using-an-env-file-recommended", + "title": "Using an env file (recommended)" }, { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#running-supabase-locally", - "title": "Running Supabase locally" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#using-inline-environment-variables", + "title": "Using inline environment variables" }, { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#updating-the-supabase-cli", - "title": "Updating the Supabase CLI" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#accessing-variables-in-functions", + "title": "Accessing variables in functions" }, { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#beta-channel", - "title": "Beta channel" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#calling-supabase-services-from-functions", + "title": "Calling Supabase services from functions" }, { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#installing-the-supabase-cli", - "title": "Installing the Supabase CLI" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#internal-vs-external-urls", + "title": "Internal vs external URLs" }, { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started#learn-more", - "title": "Learn more" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#managing-functions-via-dashboard", + "title": "Managing functions via dashboard" }, { - "url": "https://supabase.com/docs/guides/self-hosting", - "title": "Self-Hosting" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#copying-functions-from-supabase-platform", + "title": "Copying functions from Supabase platform" }, { - "url": "https://supabase.com/docs/guides/self-hosting#community-driven-projects", - "title": "Community-driven projects" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#troubleshooting", + "title": "Troubleshooting" }, { - "url": "https://supabase.com/docs/guides/self-hosting#about-self-hosting", - "title": "About self-hosting" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#500-error-on-invocation", + "title": "500 error on invocation" }, { - "url": "https://supabase.com/docs/guides/self-hosting#how-self-hosted-supabase-differs", - "title": "How self-hosted Supabase differs" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#changes-to-function-code-not-reflected-after-editing", + "title": "Changes to function code not reflected after editing" }, { - "url": "https://supabase.com/docs/guides/self-hosting#your-responsibilities-when-self-hosting", - "title": "Your responsibilities when self-hosting" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#custom-env-vars-not-available-in-functions", + "title": "Custom env vars not available in functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting#telemetry", - "title": "Telemetry" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#memory-or-timeout-errors", + "title": "Memory or timeout errors" }, { - "url": "https://supabase.com/docs/guides/self-hosting#support-and-community", - "title": "Support and community" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-oauth#provider-not-enabled-or-provider-seen-as-false-in-settings" }, { - "url": "https://supabase.com/docs/guides/self-hosting#enterprise-self-hosting", - "title": "Enterprise self-hosting" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#400-missing-function-name-in-request" }, { - "url": "https://supabase.com/docs/guides/self-hosting#get-started", - "title": "Get started" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions#401-invalid-jwt" } ], - "resultChars": 353761 + "resultChars": 400749 } ] }, @@ -18273,7 +19506,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "Meets rubric: identifies soft-delete-only root cause, implements auth user deletion plus RLS live-account gate, correctly explains JWTs remain cryptographically valid while protected data path has no post-commit window due to RLS, and correctly distinguishes publishable frontend keys from secret backend-only RLS-bypassing keys." + "judgeNotes": "Meets the core rubric: identifies soft-delete-only root cause, deletes the auth user/sessions and tightens RLS to block stale JWTs on the data path, explains the remaining JWT validity window consistently with that fix, and correctly distinguishes publishable vs secret keys and RLS bypass behavior." } ], "skills": { @@ -18284,166 +19517,87 @@ "calls": [ { "source": "search_docs", - "query": "query { keys: searchDocs(query: \"publishable key secret key legacy anon service_role RLS frontend\", limit: 5) { nodes { ... on Guide { title href content } ... on ManagementApiReference { title href content } } } sessions: searchDocs(query: \"delete user access token remains valid until expiry session sign out JWT revoke\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } delete: searchDocs(query: \"delete user admin API JWT remains valid delete account\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "query": "query { searchDocs(query: \"delete user JWT remains valid until expiry sign out sessions auth user deletion RLS\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" - }, - { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" - }, { "url": "https://supabase.com/docs/guides/auth/sessions", "title": "User sessions" }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signout" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V", - "title": "Should I set a shorter Max-Age parameter on the cookies?" - }, { "url": "https://supabase.com/docs/guides/auth/managing-user-data", "title": "User Management" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/guides/platform/sso/testing-best-practices", - "title": "SSO Testing and Best Practices" + "url": "https://supabase.com/docs/reference/swift/auth-admin-signout", + "title": "signOut()" + }, + { + "url": "https://supabase.com/docs/guides/storage/security/ownership", + "title": "Ownership" } ], - "resultChars": 230051 + "resultChars": 54592 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"API keys publishable secret key sb_publishable sb_secret RLS\", limit: 10) { nodes { ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"publishable key secret key frontend RLS bypass service_role new API keys\", limit: 5) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", - "title": "Creating a Supabase client for SSR" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { "url": "https://supabase.com/docs/guides/database/secure-data", "title": "Securing your data" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", - "title": "Build a User Management App with SvelteKit" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-solidjs", - "title": "Build a User Management App with SolidJS" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-vue-3", - "title": "Build a User Management App with Vue 3" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" } ], - "resultChars": 225824 + "resultChars": 49060 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"sign out JWT remains valid until expires revoke refresh tokens access token\", limit: 8) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"API keys publishable key anon role authenticated JWT RLS secret key service_role bypass RLS\", limit: 3) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V", - "title": "Should I set a shorter Max-Age parameter on the cookies?" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signout" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/server-side/advanced-guide", - "title": "Advanced guide" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/how-do-i-make-the-cookies-httponly-vwweFx", - "title": "How do I make the cookies HttpOnly?" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" } ], - "resultChars": 103031 + "resultChars": 45604 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"admin deleteUser delete user existing JWT access token valid until expiry\", limit: 6) { nodes { ... on ClientLibraryFunctionReference { title href content language methodName } ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"Understanding API keys publishable keys safe public components anon Postgres role secret keys service_role BYPASSRLS\", limit: 1) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions", - "title": "User sessions" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-admin-deleteuser" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" } ], - "resultChars": 33037 + "resultChars": 13714 } ] }, @@ -18498,7 +19652,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "The assistant correctly identified that orders was absent from the supabase_realtime publication despite the channel being subscribed, applied exactly ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, preserved courier_locations and RLS/policies, and did not blame or alter RLS/client/networking." + "judgeNotes": "Identified orders missing from supabase_realtime, added only public.orders to the existing publication, and preserved courier_locations, RLS, and policies." } ], "skills": { @@ -18509,7 +19663,7 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Postgres Changes add table supabase_realtime publication ALTER PUBLICATION\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"Postgres Changes add table supabase_realtime publication ALTER PUBLICATION\", limit: 3) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { @@ -18523,17 +19677,9 @@ { "url": "https://supabase.com/docs/guides/database/replication/pipelines", "title": "Set up Pipelines" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", - "title": "Backup and Restore using the CLI" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks", - "title": "Benchmarks" } ], - "resultChars": 121815 + "resultChars": 74883 } ] }, @@ -18566,17 +19712,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform as affected and described eight intermittent HTTP 503s across 07:00–12:00 UTC on 2026-04-28, noting the recurring gateway pattern." + "judgeNotes": "Identified image-transform and described eight intermittent gateway 503s across 07:00–12:00 UTC on 2026-04-28, matching the required recurring pattern." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "Attributes the 503s to the gateway/platform layer before function execution, grounded in gateway logs without corresponding Edge Function logs and nearby successful invocations on the same deployment. It also distinguishes the separate avatar-upload 500 as a runtime-level issue." + "judgeNotes": "The assistant clearly attributes the recurring image-transform 503s to the Edge Functions gateway/platform layer before function execution, not to application code. This is grounded in valid observations: gateway entries had no corresponding execution logs, nearby successful invocations occurred, and the avatar-upload 500 is distinguished as a separate function-level error." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "Recommended concrete next steps, including escalating to Supabase support with request IDs and timestamps, plus retry and alerting actions." + "judgeNotes": "The assistant recommended concrete next steps, including escalating to Supabase Support with project ref, gateway log IDs, and the incident time window, plus specific retry and mitigation actions." } ], "skills": { @@ -18588,7 +19734,7 @@ }, "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.6-no-skills/investigate-reliability-003-edge-function-5xx-correlation.json" }, { @@ -18642,7 +19788,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Diagnosed RLS deny-all due to no policies, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." + "judgeNotes": "The assistant correctly diagnosed RLS deny-all due to no policies, kept RLS enabled, and created authenticated owner-scoped SELECT and INSERT policies using auth.uid() = user_id with WITH CHECK for inserts." } ], "skills": { @@ -18650,7 +19796,36 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Row Level Security authenticated select insert auth.uid Data API table grants\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + }, + { + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" + } + ], + "resultChars": 100048 + } + ] }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", @@ -18701,7 +19876,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "Avatar migration was applied by `supabase db push --db-url \"$DB_URL\" --yes` in action #11, which succeeded and left remote migration history containing `20240220000000 | add_avatar_url`. The orphan bio migration was reconciled by adding local file `supabase/migrations/20240115000000_add_profile_bio.sql` in action #10, then running the same Supabase CLI push. No forbidden direct SQL mutation or prepared-statement workaround was seen; psql usage was read-only inspection." + "judgeNotes": "Avatar migration was applied through Supabase CLI with `supabase db push --db-url \"$DB_URL\" --yes`, which output `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` History was reconciled by adding the missing local migration file `supabase/migrations/20240115000000_add_profile_bio.sql`, after which `supabase db push` no longer reported the remote-only bio migration as blocking. No prohibited direct SQL mutation or prepared-statement reset workaround was used." } ], "skills": { @@ -18712,9 +19887,13 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI db push migration history mismatch repair deploy hosted project\", limit: 5) { nodes { __typename title href content } } }", + "query": "query { searchDocs(query: \"Supabase CLI migration deploy db push repair migration history troubleshooting\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, { "url": "https://supabase.com/docs/guides/deployment/database-migrations", "title": "Database Migrations" @@ -18723,26 +19902,22 @@ "url": "https://supabase.com/docs/reference/cli/supabase-migration-repair", "title": "Repair the migration history table" }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", - "title": "Backup and Restore using the CLI" - }, { "url": "https://supabase.com/docs/guides/local-development/database-migrations", "title": "Database migrations" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" + "url": "https://supabase.com/docs/reference/cli/supabase-db-push", + "title": "Push new migrations to the remote database" } ], - "resultChars": 96189 + "resultChars": 61657 } ] }, "prompt": "I'm trying to ship a migration to our hosted project and it's not working. Can you figure out what's wrong and get it deployed?", "promptSourcePath": "evals/resolve-database-001-migration-history-mismatch/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.6-no-skills/resolve-database-001-migration-history-mismatch.json" }, { @@ -18904,7 +20079,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user cbbc700e-775b-4489-94a7-6a2ce1ab7257, signUp returned {\"userId\":\"cbbc700e-775b-4489-94a7-6a2ce1ab7257\"}" + "notes": "db user 86ef878c-7e32-4092-831b-f50fcfc1cc22, signUp returned {\"userId\":\"86ef878c-7e32-4092-831b-f50fcfc1cc22\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -18919,7 +20094,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"cbbc700e-775b-4489-94a7-6a2ce1ab7257\"}" + "notes": "{\"userId\":\"86ef878c-7e32-4092-831b-f50fcfc1cc22\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -18947,46 +20122,7 @@ ] }, "docs": { - "calls": [ - { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 93795 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"signUp email password user metadata display_name options data\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", - "title": "Customizing Emails by Language" - }, - { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", - "title": "Passwordless email logins" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signup" - } - ], - "resultChars": 36311 - } - ] + "calls": [] }, "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", @@ -19026,7 +20162,7 @@ { "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "found 2 rows" + "notes": "found 3 rows" }, { "name": "row level security is enabled on todos", @@ -19044,7 +20180,7 @@ { "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "2 rows" + "notes": "3 rows" } ], "skills": { @@ -19114,7 +20250,19 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "web_fetch", + "query": "https://supabase.com/docs/guides/local-development/declarative-database-schemas.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas.md" + } + ], + "resultChars": 8727 + } + ] }, "prompt": "Add a description text column to the `products` table in my local Supabase stack", "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", @@ -19158,7 +20306,7 @@ { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 3) from the queue" + "notes": "function removed the seeded message (id 37) from the queue" } ], "skills": { @@ -19175,37 +20323,20 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"pgmq queue cron schedule edge function pop read delete messages\", limit: 6) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"queues pgmq cron schedule edge function pop messages\", limit: 5) { nodes { title href content } } }", "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - }, - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - } - ], - "resultChars": 68777 + "pages": [], + "resultChars": 344 }, { "source": "shell_fetch", - "query": "curl -s https://supabase.com/changelog.md | grep -i -B2 -A2 \"breaking\" | grep -i -B1 -A1 -E \"cron|pgmq|queue|edge function\" | head -40; echo \"---done---\"", + "query": "curl -s --max-time 30 https://supabase.com/changelog.md | grep -i -B2 -A2 -E \"breaking\" | grep -i -E \"cron|queue|pgmq|edge|function|breaking\" | head -20; echo \"EXIT: $?\"", "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 1492 + "resultChars": 3661 } ] }, @@ -19234,7 +20365,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { "name": "report runs and prints JSON", @@ -19253,8 +20384,8 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + "passed": true, + "notes": "imports found in: app/report.mjs" }, { "name": "report queries via the Data API, not raw SQL", @@ -19276,7 +20407,7 @@ }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "opencode-kimi-k3/build-dataapi-001-relational-report.json" }, { @@ -19299,7 +20430,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { "name": "report runs and prints JSON", @@ -19318,8 +20449,8 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + "passed": true, + "notes": "imports found in: app/restock.mjs" }, { "name": "report queries via the Data API, not raw SQL", @@ -19337,11 +20468,22 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md 2>/dev/null | head -60", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 3163 + } + ] }, "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "opencode-kimi-k3/build-dataapi-002-restock-alert-report.json" }, { @@ -19396,7 +20538,19 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "web_fetch", + "query": "https://supabase.com/docs/guides/resources/migrating-to-supabase/postgres.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/resources/migrating-to-supabase/postgres.md" + } + ], + "resultChars": 19900 + } + ] }, "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", @@ -19463,46 +20617,7 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge function verify JWT auth getUser getClaims authorization header\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-getclaims" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-getclaims", - "title": "getClaims()" - } - ], - "resultChars": 52010 - }, - { - "source": "web_fetch", - "query": "https://supabase.com/docs/guides/functions/auth.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth.md" - } - ], - "resultChars": 7275 - } - ] + "calls": [] }, "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", @@ -19542,137 +20657,72 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 401: {\"error\":\"Invalid credentials\"}" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"1f7a732a-f911-4415-80cd-89a9f4e1e75e\",\"metric\":\"steps_a_msj163uc\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"237c1242-045d-42f2-949b-5fa69b893cd8\",\"metric\":\"steps_a_msszf6r6\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: [{\"user_id\":\"1f7a732a-f911-4415-80cd-89a9f4e1e75e\",\"metric\":\"steps_a_msj163uc\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"237c1242-045d-42f2-949b-5fa69b893cd8\",\"metric\":\"steps_a_msszf6r6\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"9a247c74-daaa-4e1f-9bd6-4c24ce42abcd\",\"metric\":\"steps_b_msj163uc\",\"value\":222}]" + "notes": "status 200: [{\"user_id\":\"ee5b28a7-e005-49b5-a9a0-291b93d4837a\",\"metric\":\"steps_b_msszf6r6\",\"value\":222}]" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 401: {\"error\":\"Invalid credentials\"}" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 401: {\"error\":\"Invalid credentials\"}" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 401: {\"error\":\"Invalid credentials\"}" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { "name": "implementation uses @supabase/server", "passed": true, "notes": "imports @supabase/server / withSupabase" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 93795 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge function environment variables SUPABASE_URL SUPABASE_SERVICE_ROLE_KEY secret key publishable key\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" - } - ], - "resultChars": 94601 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"@supabase/server SDK edge functions createServerClient\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" - }, - { - "url": "https://supabase.com/docs/guides/auth/server-side/migrating-to-ssr-from-auth-helpers", - "title": "Migrating to the SSR package from Auth Helpers" - }, - { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", - "title": "Creating a Supabase client for SSR" - } - ], - "resultChars": 59816 - }, + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ { "source": "web_fetch", - "query": "https://supabase.com/docs/guides/functions/auth.md", + "query": "https://supabase.com/changelog.md", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/auth.md" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 7275 + "resultChars": 93795 } ] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/build-functions-005-dual-auth-user-secret.json" }, { @@ -19712,7 +20762,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019fdc8e-ae2d-7481-a05e-6e5700e0781e/receipt-alpha.pdf, 019fdc8e-ae2d-7481-a05e-6e5700e0781e/receipt-beta.pdf" + "notes": "saw: 01a0005e-ea79-743d-8e4d-93ec8565cc15/receipt-alpha.pdf, 01a0005e-ea79-743d-8e4d-93ec8565cc15/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -19733,7 +20783,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Meets all criteria: private user-files bucket, authenticated owner-scoped SELECT and INSERT policies with WITH CHECK, no RLS disabling or permissive/public policies, and supabase-js createSignedUrl with expiry for temporary sharing." + "judgeNotes": "Meets all criteria: creates private user-files bucket, keeps RLS enabled, defines authenticated owner-scoped SELECT and INSERT policies with WITH CHECK using user-id path scoping, and provides supabase-js createSignedUrl code with expiry for temporary sharing. No public bucket, permissive policies, public URL, anon role, or client service-role usage." } ], "skills": { @@ -19742,37 +20792,25 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"storage createSignedUrl supabase-js expiresIn private bucket\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"createSignedUrl expiring temporary download link private bucket supabase-js\") { nodes { title href content } } }", "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/storage/serving/downloads", - "title": "Serving assets from Storage" - }, - { - "url": "https://supabase.com/docs/guides/storage/serving/image-transformations", - "title": "Storage Image Transformations" - }, - { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/storage-image-transformations", - "title": "Manage Storage Image Transformations usage" - }, - { - "url": "https://supabase.com/docs/guides/storage/uploads/resumable-uploads", - "title": "Resumable Uploads" - }, - { - "url": "https://supabase.com/docs/reference/javascript/file-buckets-createsigneduploadurl" - } - ], - "resultChars": 44528 + "pages": [], + "resultChars": 345 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"storage RLS policy user own folder bucket_id storage.foldername auth.uid\") { nodes { title href content } } }", + "hasContent": true, + "pages": [], + "resultChars": 345 } ] }, @@ -19805,17 +20843,17 @@ { "name": "pgTAP test file(s) written under supabase/tests/", "passed": true, - "notes": "1 file(s): supabase/tests/tenant_isolation_test.sql" + "notes": "1 file(s): supabase/tests/tenant_isolation.sql" }, { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "3 passed, 1 failed" + "notes": "10 passed, 0 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as the table with broken tenant isolation, explains that authenticated members can read posts from orgs they are not members of, and grounds this in the pgTAP result where the `posts` negative test fails. It also correctly states `notes` is isolated and treats test results as authoritative." + "judgeNotes": "The agent correctly identifies `posts` as the table with the broken tenant isolation policy, explains that authenticated members of any org could read other organizations' posts due to missing `m.org_id = posts.org_id`, and distinguishes `notes` as correctly isolated. It grounds the conclusion in pgTAP results showing original-schema failures specifically for posts cross-tenant reads and treats the tests as authoritative." } ], "skills": { @@ -19915,42 +20953,31 @@ }, { "source": "search_docs", - "query": "{ searchDocs(query: \"semantic search pgvector match_document_sections embedding function\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"semantic search match_document_sections pgvector hnsw index RLS\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/ai/semantic-search", - "title": "Semantic search" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/semantic-search", - "title": "Semantic Search" + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" }, { "url": "https://supabase.com/docs/guides/ai/hybrid-search", "title": "Hybrid search" }, { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" + "url": "https://supabase.com/docs/guides/ai/vector-indexes", + "title": "Vector indexes" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pgvector", - "title": "pgvector: Embeddings and vector similarity" - } - ], - "resultChars": 68270 - }, - { - "source": "web_fetch", - "query": "https://supabase.com/docs/guides/ai/semantic-search.md", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" + }, { - "url": "https://supabase.com/docs/guides/ai/semantic-search.md" + "url": "https://supabase.com/docs/guides/ai/vector-indexes/ivf-indexes", + "title": "IVFFlat indexes" } ], - "resultChars": 12827 + "resultChars": 46448 } ] }, @@ -19976,7 +21003,7 @@ "observability" ], "suite": "benchmark", - "passed": true, + "passed": false, "checks": [ { "name": "preserved existing app scrape job", @@ -19984,13 +21011,13 @@ }, { "name": "configured the Supabase Metrics API scrape correctly", - "passed": true, - "judgeNotes": "Meets all requirements: HTTPS Supabase Metrics API scrape for the project target, correct metrics path, Basic Auth with password_file, app scrape preserved, and docker-compose mounts the secrets directory containing the password file." + "passed": false, + "judgeNotes": "Supabase scrape preserves app job, uses correct metrics_path and basic_auth with password_file, and mounts secrets. However it is not deployable for a hosted Supabase project: scheme is http and target is host.docker.internal:43237 instead of .supabase.co:443 or .supabase.red:443 over HTTPS." }, { "name": "documented live deployment and verification steps", "passed": true, - "judgeNotes": "README includes Secret API key creation, matching secret file placement, stack restart/reload, and concrete verification via Prometheus targets plus direct curl check. Endpoint/auth and mounted secret path are consistent, with no hardcoded secret." + "judgeNotes": "README includes Secret API key creation, matching password_file placement, Compose restart/reload, and concrete verification via Prometheus targets/PromQL. Endpoint/auth and secret setup match the provided config." } ], "skills": { @@ -20005,53 +21032,88 @@ "docs": { "calls": [ { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", + "source": "search_docs", + "query": "{ searchDocs(query: \"metrics endpoint prometheus scrape monitoring\", limit: 10) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", + "title": "Metrics API" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", + "title": "Metrics API with Prometheus & Grafana (self-hosted)" + }, + { + "url": "https://supabase.com/docs/reference/api/v1-scrape-project-metrics", + "title": "Scrape a project's metrics" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" + }, + { + "url": "https://supabase.com/docs/guides/database/replication/pipelines-monitoring", + "title": "Monitor pipeline status" } ], - "resultChars": 93795 + "resultChars": 47306 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"metrics API prometheus endpoint scrape\") { nodes { title href content } } }", + "query": "{ searchDocs(query: \"management api create project api key secret\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", - "title": "Metrics API with Prometheus & Grafana (self-hosted)" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" + "url": "https://supabase.com/docs/reference/api/v1-create-project-api-key", + "title": "Creates a new API key for the project" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" + "url": "https://supabase.com/docs/reference/api/v1-create-a-project", + "title": "Create a project" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/how-to-view-database-metrics-uqf2z_", - "title": "How to View Database Metrics" + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" }, { - "url": "https://supabase.com/docs/guides/platform/read-replicas", - "title": "Read Replicas" + "url": "https://supabase.com/docs/reference/api/v1-get-project-api-key", + "title": "Get API key" + } + ], + "resultChars": 97072 + }, + { + "source": "shell_fetch", + "query": "curl -s --max-time 20 https://supabase.com/changelog.md | grep -i -B2 -A5 \"metric\" | head -60", + "pages": [ + { + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 29060 + "resultChars": 1411 } ] }, "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3/deploy-database-001-prometheus-metrics.json" }, { @@ -20086,7 +21148,7 @@ { "name": "the weather function reads WEATHER_API_KEY from the environment", "passed": true, - "judgeNotes": "The function reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." + "judgeNotes": "Reads WEATHER_API_KEY from the runtime environment using Deno.env.get(\"WEATHER_API_KEY\")." }, { "name": "WEATHER_API_KEY value is not committed to the repo", @@ -20103,7 +21165,18 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md | grep -i -B2 -A2 'breaking' | grep -i -E 'function|secret|edge|deploy' | head -20", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 1364 + } + ] }, "prompt": "Our weather widget currently calls WeatherAPI straight from the browser, which\nleaks our API key. I want to move that behind a Supabase Edge Function called\n`weather` that holds the key server-side and proxies the request.\n\nThe function should read the key from an environment variable named\n`WEATHER_API_KEY`. Our key already lives in a local `.env` file at the project\nroot.\n\nDeploy the function to our project so it's live, and make sure the deployed\nfunction can actually read the key at runtime.", "promptSourcePath": "evals/deploy-functions-001-edge-function-secrets/PROMPT.md", @@ -20162,25 +21235,25 @@ "calls": [ { "source": "web_fetch", - "query": "https://supabase.com/docs/guides/self-hosting/docker.md", + "query": "https://supabase.com/changelog.md", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/self-hosting/docker.md" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 30709 + "resultChars": 51147 }, { "source": "web_fetch", - "query": "https://supabase.com/changelog.md", + "query": "https://supabase.com/docs/guides/self-hosting/docker.md", "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/self-hosting/docker.md" } ], - "resultChars": 93795 + "resultChars": 30885 } ] }, @@ -20238,7 +21311,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "The answer diagnoses the soft-delete-only bug, implements real auth user/session/refresh-token removal, explains the remaining stale JWT access-token window consistently with its fix (including mitigation), and correctly distinguishes publishable frontend keys from secret/server keys that bypass RLS." + "judgeNotes": "Meets all rubric requirements: diagnoses soft-delete-only flow, implements auth user deletion/session and refresh-token revocation, accurately explains stateless JWT residual window with mitigations consistent with the fix, and correctly distinguishes publishable vs secret keys and RLS behavior." } ], "skills": { @@ -20247,39 +21320,83 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { "calls": [ + { + "source": "web_fetch", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 51147 + }, { "source": "search_docs", - "query": "{ searchDocs(query: \"publishable key secret key anon service_role API keys RLS\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"delete user account self-service postgres function auth.users cascade sessions\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/guides/auth/auth-anonymous", + "title": "Anonymous Sign-Ins" }, { - "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", - "title": "Creating a Supabase client for SSR" + "url": "https://supabase.com/docs/guides/troubleshooting/dashboard-errors-when-managing-users-N1ls4A", + "title": "Errors when creating / updating / deleting users" }, { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/reference/javascript/auth-admin-deleteuser" + } + ], + "resultChars": 26901 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"publishable secret API keys anon service_role migration RLS\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" }, { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", + "title": "Build a Social Auth App with Expo React Native" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", + "title": "Build a User Management App with SvelteKit" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + } + ], + "resultChars": 169611 + }, + { + "source": "web_fetch", + "query": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys.md" } ], - "resultChars": 101422 + "resultChars": 13042 } ] }, @@ -20333,7 +21450,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Identified orders missing from supabase_realtime publication as root cause, added public.orders to the existing publication, and preserved courier_locations, RLS, and policies. Did not blame client/RLS/networking as root cause or weaken security." + "judgeNotes": "The assistant correctly identified orders missing from the supabase_realtime publication as the root cause, explained SUBSCRIBED without events, applied ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, preserved courier_locations/RLS/policies, and did not blame or weaken RLS or make client-only changes." } ], "skills": { @@ -20342,7 +21459,8 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { @@ -20371,65 +21489,36 @@ ], "suite": "benchmark", "interface": "mcp", - "passed": false, + "passed": true, "checks": [ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Identified image-transform as the affected function and explicitly described the recurring 8 HTTP 503 gateway failures across the morning of 2026-04-28 from 07:00Z to 12:00Z." + "judgeNotes": "The assistant identified `image-transform` as the affected function and described the recurring pattern of 8 intermittent HTTP 503 gateway failures across the morning of 2026-04-28, including timestamps from 07:00Z–12:00Z. It also correctly distinguished the older billing-webhook 503s as unrelated." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": false, - "judgeNotes": "Although it cites a valid observation (503s only in gateway logs with no function execution logs), it ultimately attributes the primary cause to the function's unpinned npm dependency/worker boot and recommends pinning/redeploying the functions, rather than attributing the 503s to the gateway/platform layer in front of the function." + "passed": true, + "judgeNotes": "The assistant clearly attributes the recurring image-transform 503s to the gateway/platform layer, not function code. It grounds this in valid observations: 503s only in gateway logs with no matching function execution logs, nearby function invocations succeeded, same deployment/version all morning, and distinguishes these gateway 503s from the avatar-upload function-level 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended concrete next steps: pinning Edge Function dependencies, vendoring dependencies with lockfile checks, checking npm publish/registry history for the affected time window, and adding gateway 5xx alerting." + "judgeNotes": "The assistant recommended concrete next steps, including escalating to Supabase support with gateway request IDs/timestamps, checking platform/region status, adding gateway 5xx monitoring, and client retries." } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge functions 503 error npm dependencies pin version deno.json import map cold start\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-dependency-analysis", - "title": "Edge Function dependency analysis" - }, - { - "url": "https://supabase.com/docs/guides/security/npm-security", - "title": "Securing npm installs" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/importing-stripe-or-other-modules-from-esmsh-on-deno-edge-functions-throws-an-error-TmbB5p", - "title": "Importing Stripe or other modules from esm.sh on Deno Edge Functions throws an error" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", - "title": "Building an MCP Server with mcp-lite" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - } - ], - "resultChars": 43609 - } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" ] }, + "docs": { + "calls": [] + }, "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", "promptSourcePath": "evals/investigate-reliability-003-edge-function-5xx-correlation/PROMPT.md", "attempts": 2, @@ -20485,7 +21574,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies as the cause of empty Data API results, kept RLS enabled, and created authenticated SELECT and INSERT owner-scoped policies using auth.uid(), including WITH CHECK for inserts." + "judgeNotes": "Diagnosed RLS default-deny due to no policies, created authenticated SELECT and INSERT owner-scoped policies using auth.uid() with WITH CHECK for inserts, and did not disable RLS." } ], "skills": { @@ -20549,7 +21638,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "PASS: The avatar_url migration was applied through the Supabase CLI with `supabase db push` in action #26, which shows `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration history was reconciled by adding the missing local migration file `supabase/migrations/20240115000000_add_bio.sql` in actions #22-#24, after which `supabase migration list` (#25/#28) showed local and remote history aligned. Read-only psql inspections were used, but no prohibited direct SQL mutation or prepared-statement workaround was seen." + "judgeNotes": "PASS: The pending avatar_url migration was applied by `supabase db push` in action #17, with output showing `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration history was reconciled by adding the missing local migration file `supabase/migrations/20240115000000_add_profile_bio.sql` in action #16, then letting `supabase db push` reconcile local/remote history. Only read-only psql inspection was used; no disallowed workaround or direct SQL mutation was seen." } ], "skills": { @@ -20558,8 +21647,7 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { @@ -20619,7 +21707,6 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", "supabase-postgres-best-practices" ] }, @@ -20739,7 +21826,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 21c579b2-ae12-44ca-83f7-24fe17e54af5, signUp returned {\"userId\":\"21c579b2-ae12-44ca-83f7-24fe17e54af5\"}" + "notes": "db user f5ba0e43-89d8-42c5-b3c6-e6d492ff570c, signUp returned {\"userId\":\"f5ba0e43-89d8-42c5-b3c6-e6d492ff570c\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -20754,7 +21841,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"21c579b2-ae12-44ca-83f7-24fe17e54af5\"}" + "notes": "{\"userId\":\"f5ba0e43-89d8-42c5-b3c6-e6d492ff570c\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -20937,7 +22024,7 @@ { "name": "process-tasks function drains the queue", "passed": true, - "notes": "function removed the seeded message (id 5) from the queue" + "notes": "function removed the seeded message (id 3) from the queue" } ], "skills": { @@ -20948,57 +22035,19 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"pgmq queues consume messages with edge function read delete\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"consume queue messages edge function pgmq cron\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", "title": "Consuming Supabase Queue Messages with Edge Functions" }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - }, { "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - } - ], - "resultChars": 68440 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"expose queues PostgREST edge function pgmq_public read delete example\", limit: 3) { nodes { title href subsections { nodes { title content } } } } }", - "hasContent": true, - "pages": [] - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"expose queues PostgREST edge function pgmq_public read delete example\", limit: 3) { nodes { title href ... on Guide { subsections { nodes { title content } } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" - }, - { - "url": "https://supabase.com/docs/guides/queues/expose-self-hosted-queues", - "title": "Expose Queues for local and self-hosted Supabase" } ], - "resultChars": 22663 + "resultChars": 34849 } ] }, @@ -21275,37 +22324,37 @@ { "name": "rejects request with no credentials", "passed": true, - "notes": "status 401: {\"error\":\"Invalid credentials\"}" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"8de5d728-6f94-47d9-833a-f4104a74539f\",\"metric\":\"steps_a_msj15kas\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"4d2a2ecb-ad3c-46bb-af24-52e0a95f42df\",\"metric\":\"steps_a_msszbqam\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: [{\"user_id\":\"8de5d728-6f94-47d9-833a-f4104a74539f\",\"metric\":\"steps_a_msj15kas\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"4d2a2ecb-ad3c-46bb-af24-52e0a95f42df\",\"metric\":\"steps_a_msszbqam\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"b33c6f83-e1ca-49a7-8520-64ff05e415e9\",\"metric\":\"steps_b_msj15kas\",\"value\":222}]" + "notes": "status 200: [{\"user_id\":\"da2369e6-23bd-4722-8ece-376e7048a0a4\",\"metric\":\"steps_b_msszbqam\",\"value\":222}]" }, { "name": "non-service key is not granted service access", "passed": true, - "notes": "status 401: {\"error\":\"Invalid credentials\"}" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { "name": "rejects an unverified (forged) user token", "passed": true, - "notes": "status 401: {\"error\":\"Invalid credentials\"}" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { "name": "a user token in the apikey slot is not treated as the service key", "passed": true, - "notes": "status 401: {\"error\":\"Invalid credentials\"}" + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" }, { "name": "implementation uses @supabase/server", @@ -21321,136 +22370,89 @@ "calls": [ { "source": "search_docs", - "query": "{searchDocs(query: \"edge function verify JWT get user from access token service role bypass RLS\") {nodes {title href content}}}", + "query": "{\n searchDocs(query: \"edge function verify service role key apikey header authorization\", limit: 8) {\n nodes {\n title\n href\n content\n }\n }\n}", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", - "title": "Why is my service role key client getting RLS errors or not returning data?" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac", - "title": "Custom Claims & Role-based Access Control (RBAC)" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" }, { - "url": "https://supabase.com/docs/guides/auth/jwts", - "title": "JSON Web Token (JWT)" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" }, { "url": "https://supabase.com/docs/guides/functions/auth", "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", - "title": "Storage Buckets" - }, - { - "url": "https://supabase.com/docs/guides/auth/users", - "title": "Users" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-getclaims" + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" }, { - "url": "https://supabase.com/docs/reference/swift/auth-getclaims", - "title": "getClaims()" - }, + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + } + ], + "resultChars": 118701 + }, + { + "source": "search_docs", + "query": "{\n searchDocs(query: \"edge functions environment variables publishable secret API keys SUPABASE_PUBLISHABLE_KEY\", limit: 8) {\n nodes {\n title\n href\n content\n }\n }\n}", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/getting-started", - "title": "Getting Started with OAuth 2.1 Server" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-select-returning-an-empty-data-array-and-i-have-data-in-the-table-xvOPgx", - "title": "Why is my select returning an empty data array and I have data in the table?" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" }, { - "url": "https://supabase.com/docs/guides/storage/security/ownership", - "title": "Ownership" + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" }, { - "url": "https://supabase.com/docs/reference/swift/auth-getuser", - "title": "user()" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0012_auth_allow_anonymous_sign_ins", - "title": "Database Advisor: Lint 0012_auth_allow_anonymous_sign_ins" + "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", + "title": "Build a Social Auth App with Expo React Native" }, { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" } ], - "resultChars": 271984 + "resultChars": 167096 } ] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "opencode-kimi-k3-no-skills/build-functions-005-dual-auth-user-secret.json" }, { @@ -21490,7 +22492,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019fdc8d-4da9-7774-82e4-97e7592afbcb/receipt-alpha.pdf, 019fdc8d-4da9-7774-82e4-97e7592afbcb/receipt-beta.pdf" + "notes": "saw: 01a0005e-a02a-746b-ad46-0db5dec107a4/receipt-alpha.pdf, 01a0005e-a02a-746b-ad46-0db5dec107a4/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -21511,7 +22513,7 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Meets the rubric: creates a private user-files bucket, adds authenticated owner-scoped SELECT and INSERT policies on storage.objects using the user-id path segment, does not disable RLS or use public access, and provides supabase-js createSignedUrl code with an expiry." + "judgeNotes": "Meets the rubric: private user-files bucket, authenticated owner-scoped SELECT and INSERT policies on storage.objects, no RLS disabling or public access, and supabase-js createSignedUrl with expiry for temporary sharing." } ], "skills": { @@ -21555,12 +22557,12 @@ { "name": "pgTAP isolation tests ran and pass", "passed": true, - "notes": "10 passed, 2 failed" + "notes": "10 passed, 3 failed" }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, - "judgeNotes": "The agent correctly identifies `posts` as the broken tenant isolation policy, explains that authenticated members can read posts from organizations they do not belong to, and grounds the conclusion in pgTAP failures. It also correctly notes `notes` is isolated and treats the test results as authoritative." + "judgeNotes": "The agent correctly identifies `posts` as having the broken tenant isolation policy, specifically that authenticated members can read posts from organizations they are not members of. It grounds this in the pgTAP failures and distinguishes `notes` as passing. It also mentions an additional memberships issue, but that does not undermine the required conclusion." } ], "skills": { @@ -21634,7 +22636,36 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"semantic search pgvector match_document_sections gte-small embedding column\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + }, + { + "url": "https://supabase.com/docs/guides/ai/hybrid-search", + "title": "Hybrid search" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pgvector", + "title": "pgvector: Embeddings and vector similarity" + }, + { + "url": "https://supabase.com/docs/guides/ai/vector-columns", + "title": "Vector columns" + } + ], + "resultChars": 70040 + } + ] }, "prompt": "We're adding semantic search to our internal knowledge base app. I already wrote the edge functions, but search doesn't work yet. Can you set up whatever the database needs to make search work end to end?\n\nSome documents are confidential, so users should only have access to documents they own.", "promptSourcePath": "evals/build-vectors-001-rag-with-permissions/PROMPT.md", @@ -21667,12 +22698,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": true, - "judgeNotes": "Meets all requirements: app scrape preserved, Supabase HTTPS metrics endpoint configured with Basic Auth password_file, project target present, and docker-compose mounts the secrets directory containing the password file." + "judgeNotes": "Prometheus preserves the app scrape and adds a Supabase scrape using HTTPS, the required metrics path, Basic Auth with password_file, and a valid .supabase.co target. docker-compose mounts the secrets directory containing the password file into the Prometheus container." }, { "name": "documented live deployment and verification steps", "passed": true, - "judgeNotes": "README includes correct live setup steps: create/copy a Supabase Secret API key, write it to the mounted secret file matching prometheus.yml, restart/recreate or reload the Compose stack, and verify via Prometheus targets and PromQL/Grafana. Endpoint/auth and secret setup are consistent, with no hardcoded real secret." + "judgeNotes": "README includes creating/copying a Supabase Secret API key, placing it in the mounted secret file, applying via compose up or Prometheus reload, and verifying with Prometheus targets/PromQL/Grafana. No hardcoded secret or mismatched setup." } ], "skills": { @@ -21683,53 +22714,33 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"metrics endpoint prometheus scrape hosted project observability\", limit: 5) { nodes { title href content } } }", + "query": "{\n searchDocs(query: \"prometheus metrics endpoint scrape Supabase project\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", - "title": "Metrics API" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", - "title": "Vendor-agnostic Metrics API setup" - }, { "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-self-hosted", "title": "Metrics API with Prometheus & Grafana (self-hosted)" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", - "title": "Metrics API with Grafana Cloud" - } - ], - "resultChars": 19334 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"management api create project api key secret sb_secret\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics", + "title": "Metrics API" }, { - "url": "https://supabase.com/docs/reference/api/v1-create-project-api-key", - "title": "Creates a new API key for the project" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/grafana-cloud", + "title": "Metrics API with Grafana Cloud" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging/metrics/vendor-agnostic", + "title": "Vendor-agnostic Metrics API setup" } ], - "resultChars": 63685 + "resultChars": 19334 } ] }, "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nin the observability README what we need to do to make the config live?", "promptSourcePath": "evals/deploy-database-001-prometheus-metrics/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "opencode-kimi-k3-no-skills/deploy-database-001-prometheus-metrics.json" }, { @@ -21883,7 +22894,7 @@ { "name": "diagnosed and explained session revocation", "passed": true, - "judgeNotes": "The answer identifies the soft-delete-only root cause, implements real auth/session/refresh-token revocation by deleting sessions and auth user, hardens RLS to close the data API stale-JWT window, and explains the remaining stateless JWT validity caveat consistently. It also correctly distinguishes publishable frontend keys with RLS from secret server-only keys that bypass RLS." + "judgeNotes": "Meets all rubric criteria: diagnoses soft-delete-only flow, implements auth.users deletion with session/refresh-token revocation via cascade, consistently explains remaining stateless JWT access window and mitigation, and correctly distinguishes publishable frontend/RLS behavior from secret server-only/RLS-bypass behavior." } ], "skills": { @@ -21891,7 +22902,28 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"publishable secret API keys vs anon service_role migration\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/rotating-anon-service-and-jwt-secrets-1Jq6yd", + "title": "Rotating Anon, Service, and JWT Secrets" + } + ], + "resultChars": 46739 + } + ] }, "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", @@ -21943,7 +22975,7 @@ { "name": "diagnosed missing publication membership", "passed": true, - "judgeNotes": "Identifies orders missing from supabase_realtime publication as root cause despite SUBSCRIBED status, applies ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, verifies existing courier_locations remains, and does not weaken RLS/policies or blame client/networking." + "judgeNotes": "The assistant correctly identified the root cause as orders missing from the supabase_realtime publication, applied ALTER PUBLICATION supabase_realtime ADD TABLE public.orders, and did not weaken RLS/policies or disrupt courier_locations." } ], "skills": { @@ -21981,17 +23013,17 @@ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "The assistant clearly identified image-transform as the affected function and described the recurring pattern of 8 HTTP 503 gateway responses on 2026-04-28 between 07:00Z and 12:00Z, while correctly distinguishing unrelated old billing-webhook 503s." + "judgeNotes": "The assistant correctly identified `image-transform` as the affected function and described the recurring pattern of 8 HTTP 503 gateway failures across the morning of 2026-04-28 from 07:00Z to 12:00Z, while distinguishing it from the older billing-webhook issue." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", "passed": true, - "judgeNotes": "Attributes the 503s to the gateway/edge platform layer, not function code, and grounds this in valid evidence: gateway-only 503s with no failed function invocations, successful nearby executions on same deployment/version, and distinction from a function-level avatar-upload 500." + "judgeNotes": "Attributes recurring image-transform 503s to the gateway/platform layer, explicitly not function code. Grounds this in valid observations: 503s only in API gateway logs with no corresponding edge function execution rows, nearby executed requests succeeded quickly, and distinguishes gateway 503s from avatar-upload's function-level 500." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended concrete next steps, including opening a Supabase support ticket with project ref, request IDs, and time window, checking platform incident/status, adding retries, and setting up alerting." + "judgeNotes": "The assistant recommended concrete next steps including vendoring/pinning the dependency, adding retries, escalating to Supabase support with specific gateway request IDs and timestamps, and setting up monitoring for gateway 5xx without function executions." } ], "skills": { @@ -22056,7 +23088,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "The assistant correctly diagnosed RLS enabled with no policies causing deny-all Data API results, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to auth.uid() = user_id using USING and WITH CHECK." + "judgeNotes": "Diagnosed missing RLS policies as the cause, kept RLS enabled, and created authenticated-only owner-scoped SELECT and INSERT policies using auth.uid() with WITH CHECK for inserts." } ], "skills": { @@ -22114,7 +23146,7 @@ { "name": "the avatar migration and history reconciliation were done via the Supabase CLI", "passed": true, - "judgeNotes": "Avatar_url was applied through `supabase db push` in #17, which shows `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` The orphan bio migration was reconciled by adding the missing local migration file `20240115000000_add_profile_bio.sql` in #15, after which Supabase CLI migration list showed local and remote aligned. No prohibited workaround or direct mutation was used." + "judgeNotes": "Applied pending avatar_url migration with `supabase db push` in #16; output shows `Applying migration 20240220000000_add_avatar_url.sql...` and `Finished supabase db push.` Reconciled orphan bio migration by adding local file `supabase/migrations/20240115000000_add_profile_bio.sql` in #15, then `supabase db push` in #16 allowed remote/local history to align, confirmed by `supabase migration list` in #18/#22. Only read-only psql inspection was used; no disallowed workaround seen." } ], "skills": { @@ -22165,7 +23197,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=79.74..79.86 rows=50 width=58)\n -> Sort (cost=79.74..79.99 rows=100 width=58)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=5.06..76.42 rows=100 width=58)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on idx_events_user_created_at (cost=0.00..5.03 rows=100 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index f8ca77b3..5f87100b 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -32,7 +32,7 @@ { "name": "suggested relevant next steps for a Supabase + Next.js project", "passed": true, - "judgeNotes": "The response provides multiple concrete Supabase/Next.js next steps: running `supabase start`, checking Docker, installing `@supabase/supabase-js`/`@supabase/ssr`, setting `.env.local` URL and anon key, linking a remote project, and creating a first migration." + "judgeNotes": "The response provides concrete Supabase/Next.js next steps: running `supabase start`, linking a project, installing `@supabase/supabase-js`, setting `.env.local` with `NEXT_PUBLIC_SUPABASE_URL` and anon key, adding gitignore entries, and creating migrations." }, { "name": "attempted `npx plugins add supabase-community/supabase-plugin`", @@ -90,17 +90,17 @@ { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"24ebdf94-3b1a-4f68-92ed-d030e5f27f7f\",\"metric\":\"steps_a_msr5rarn\",\"value\":111}]}" + "notes": "status 200: [{\"user_id\":\"c5d28067-ff2e-430b-97fd-30724feb48f4\",\"metric\":\"steps_a_mssymbd6\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"24ebdf94-3b1a-4f68-92ed-d030e5f27f7f\",\"metric\":\"steps_a_msr5rarn\",\"value\":111}]}" + "notes": "status 200: [{\"user_id\":\"c5d28067-ff2e-430b-97fd-30724feb48f4\",\"metric\":\"steps_a_mssymbd6\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"b23c8b7b-030c-45e9-bc10-ca4e3c8c88b4\",\"metric\":\"steps_b_msr5rarn\",\"value\":222}]}" + "notes": "status 200: [{\"user_id\":\"0ff50323-c5ee-421d-a405-07c2eb1d5f4c\",\"metric\":\"steps_b_mssymbd6\",\"value\":222}]" }, { "name": "non-service key is not granted service access", @@ -136,7 +136,7 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"@supabase/server withSupabase edge function multi-auth\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"@supabase/server withSupabase edge function multi-auth\", limit: 10) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -158,9 +158,29 @@ { "url": "https://supabase.com/docs/guides/auth/enterprise-sso", "title": "Enterprise Single Sign-On" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", + "title": "Resumable WebSockets with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" + }, + { + "url": "https://supabase.com/docs/guides/ai-tools/byo-mcp", + "title": "Deploy MCP servers" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" } ], - "resultChars": 32262 + "resultChars": 74670 } ] }, @@ -198,7 +218,7 @@ { "name": "did not recommend read replicas for Realtime", "passed": true, - "judgeNotes": "The assistant correctly treats the request as Supabase Realtime/Postgres Changes setup, adds the messages table to the supabase_realtime publication, provides a postgres_changes client subscription, and does not recommend or confuse read replicas." + "judgeNotes": "The assistant correctly treated the task as Supabase Realtime/Postgres Changes setup, added the table to the supabase_realtime publication, verified it, and provided a postgres_changes client subscription example. It did not recommend or imply read replicas are required or useful for live updates." } ], "skills": { @@ -211,7 +231,36 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"enable realtime postgres changes on a table publication\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", + "title": "Subscribing to Database Changes" + }, + { + "url": "https://supabase.com/docs/guides/realtime/postgres-changes", + "title": "Postgres Changes" + }, + { + "url": "https://supabase.com/docs/guides/realtime/benchmarks", + "title": "Benchmarks" + }, + { + "url": "https://supabase.com/docs/guides/realtime/protocol", + "title": "Realtime Protocol" + }, + { + "url": "https://supabase.com/docs/guides/realtime/authorization", + "title": "Realtime Authorization" + } + ], + "resultChars": 112153 + } + ] }, "prompt": "I'm building a simple chat app on Supabase.\n\nUsers can send messages, and I want everyone in the same room to see new\nmessages appear automatically without refreshing the page.\n\nCan you inspect the project and set up whatever Supabase needs for live updates?", "promptSourcePath": "evals/build-realtime-001-live-chat-updates/PROMPT.md", @@ -241,17 +290,17 @@ { "name": "identified video-thumbnails and the 546 resource-limit pattern", "passed": true, - "judgeNotes": "Identified video thumbnail generation as the affected function/workload and correctly recognized HTTP 546 WORKER_LIMIT/resource-limit CPUTime errors, not 500/503 server errors." + "judgeNotes": "Identified `video-thumbnails` as affected and correctly described failures as HTTP 546 due to CPU/resource limit, not 500/503." }, { "name": "attributed the 546s to CPU time exhaustion", "passed": true, - "judgeNotes": "The assistant explicitly attributes the 546 failures to CPU time exhaustion, citing log evidence: shutdown reason CPUTime and cpu_time_used 2000ms / cpu_time_limit 2000ms. It distinguishes this from wall-clock time, memory, exceptions, or unrelated errors." + "judgeNotes": "The assistant specifically attributed the 546s to CPU time exhaustion, citing shutdown reason `CPUTime` and `cpu_time_used: 2000ms / limit: 2000ms`, and distinguished it from wall-clock/memory/other errors." }, { "name": "recommended reducing/offloading CPU work as the fix", "passed": true, - "judgeNotes": "The assistant correctly identified the CPUTime limit and recommended reducing CPU work per request, optimizing thumbnail generation, and moving transcoding to a background/external worker/service rather than retrying or raising limits." + "judgeNotes": "The assistant clearly recommends reducing/offloading CPU-intensive thumbnail generation: enqueue work, move transcoding/frame extraction to an external worker/service, and optimize/constrain decoding as a short-term mitigation. It also correctly avoids suggesting timeouts or plan upgrades as the fix." } ], "skills": { @@ -267,31 +316,31 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"edge function CPU time limit shutdown 546\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"edge function CPU time limit worker shutdown\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-546-error-response", - "title": "546 - WORKER_RESOURCE_LIMIT Exceeded / WORKER_LIMIT Exceeded" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-wall-clock-time-limit-reached-Nk38bW", - "title": "Edge Function 'wall clock time limit reached'" + "url": "https://supabase.com/docs/guides/troubleshooting/edge-functions-worker-timeouts-and-websocket-drops", + "title": "Edge Functions worker timeouts and WebSocket drops" }, { "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-cpu-limits", "title": "Understanding Edge Function CPU limits" }, { - "url": "https://supabase.com/docs/guides/functions/limits", - "title": "Limits" + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-shutdown-reasons-explained", + "title": "Edge Function shutdown reasons explained" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-wall-clock-time-limit-reached-Nk38bW", + "title": "Edge Function 'wall clock time limit reached'" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-504-error-response", - "title": "Edge Function 504 error response" + "url": "https://supabase.com/docs/guides/functions/limits", + "title": "Limits" } ], - "resultChars": 28230 + "resultChars": 24082 } ] }, @@ -367,7 +416,7 @@ { "name": "diagnosed secure default grants without weakening RLS", "passed": true, - "judgeNotes": "Diagnoses secure-by-default/Data API explicit grant issue, distinguishes grants from RLS, preserves owner-scoped RLS, grants SELECT/INSERT only to authenticated, and verifies behavior without granting anon/public or weakening policies." + "judgeNotes": "The answer correctly diagnoses missing Data API/Postgres grants under secure-by-default behavior, distinguishes grants from RLS, preserves owner-scoped RLS, grants only SELECT and INSERT on public.journal_entries to authenticated, and does not weaken RLS or grant anon/public." } ], "skills": { @@ -415,7 +464,7 @@ { "name": "user A's update actually checks off their own task", "passed": true, - "notes": "saw: [{\"id\":\"e6664c25-a1a6-4c61-8097-cd7bbb62626c\",\"is_done\":true}]" + "notes": "saw: [{\"id\":\"2c721c11-4599-4e0a-b6bf-4fec1c1f9a4f\",\"is_done\":true}]" }, { "name": "user B cannot update user A's task", @@ -424,7 +473,7 @@ { "name": "diagnosed the missing USING clause and added it", "passed": true, - "judgeNotes": "Diagnosed the zero-row UPDATE as the tasks UPDATE RLS policy missing a USING clause, explained WITH CHECK vs row targeting, and fixed it with an authenticated-only ownership-scoped USING plus retained WITH CHECK on user_id = auth.uid(). RLS was not disabled and the fix was verified." + "judgeNotes": "The assistant correctly diagnosed the zero-row UPDATE as caused by the UPDATE RLS policy missing a USING clause, explained that WITH CHECK alone does not select existing rows for update, and fixed it by recreating the authenticated-only policy with USING scoped to user_id = auth.uid() while preserving WITH CHECK to prevent reassignment. RLS was not disabled and isolation was verified." } ], "skills": { @@ -437,7 +486,28 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"row level security policy USING WITH CHECK UPDATE missing using clause\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0024_permissive_rls_policy", + "title": "Database Advisor: Lint 0024_permissive_rls_policy" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" + } + ], + "resultChars": 54114 + } + ] }, "prompt": "Our app lets signed-in users manage a personal `tasks` list. Users can create tasks and check them off (`is_done`).\n\nCreating a task works fine, and I can see the row in the table. But when a user checks off a task, the app's update call succeeds with no error, yet `is_done` never actually changes, and the API doesn't return the updated row either.\n\nFind out why the update has no effect and fix it.", "promptSourcePath": "evals/resolve-dataapi-002-update-zero-rows-affected/PROMPT.md", @@ -467,7 +537,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "Answer correctly says pause/restore is not the right recovery path, recommends restart as first step, distinguishes restore from restart/backups, suggests checking resource pressure and scaling/tuning, and escalating to support if recurring. Minor omission: could have mentioned logs/advisors/troubleshooting docs more explicitly, but guidance is concrete and safe." + "judgeNotes": "The answer clearly says pause/restore is not the right recovery path and recommends restart first. It distinguishes restart from pause/restore and gives concrete next steps like checking reports/advisors/logs and scaling/resource investigation if it recurs. It could mention contacting Supabase support and waiting a few minutes more explicitly, but overall meets the rubric." } ], "skills": { @@ -581,7 +651,46 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md | head -100", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 5516 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"publishable key secret key migrate anon key service_role API keys\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", + "title": "Supabase for Platforms" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" + } + ], + "resultChars": 107177 + } + ] }, "prompt": "Heads-up from the platform team: the legacy JWT-based API keys (`anon` /\n`service_role`) are going away for our projects soon, in favor of the new\npublishable/secret keys. The little blog tooling app in `app/` still uses the\nlegacy keys.\n\nMigrate it over. Both scripts need to keep working — `npm run posts` and\n`npm run stats` (run them from `app/`). The local Supabase project in\n`supabase/` is already running.", "promptSourcePath": "evals/resolve-sdk-001-legacy-key-migration/PROMPT.md", @@ -629,7 +738,7 @@ { "name": "user A can replace their own avatar via upsert", "passed": true, - "notes": "saw: [{\"name\":\"019ff9df-beb8-74c8-b84b-21f4e91826eb/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" + "notes": "saw: [{\"name\":\"01a0005e-2b22-74be-9ce0-f0e8e4ce6bbb/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" }, { "name": "user B cannot overwrite user A's avatar", @@ -638,7 +747,7 @@ { "name": "added an owner-scoped UPDATE policy without weakening public reads", "passed": true, - "judgeNotes": "The answer correctly diagnoses missing UPDATE RLS policy on storage.objects for upsert replacements, notes public bucket only affects read/download behavior, keeps bucket public/RLS enabled, and adds an authenticated owner-scoped UPDATE policy with USING and WITH CHECK based on path user id." + "judgeNotes": "Diagnosed missing UPDATE RLS policy for upsert replacement, noted public bucket only affects reads/download URLs, kept bucket public/RLS enabled, and added an authenticated owner-scoped UPDATE policy with USING and WITH CHECK." } ], "skills": { @@ -646,7 +755,9 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [] + "loaded": [ + "supabase" + ] }, "docs": { "calls": [] @@ -695,17 +806,17 @@ { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"4ddfbb4b-389b-48ab-a2e9-5379f47338e3\",\"metric\":\"steps_a_msr5t2ph\",\"value\":111}]}" + "notes": "status 200: {\"data\":[{\"user_id\":\"3788dfbd-cd15-42c6-b452-c86a06f1d822\",\"metric\":\"steps_a_mssyktn2\",\"value\":111}]}" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"4ddfbb4b-389b-48ab-a2e9-5379f47338e3\",\"metric\":\"steps_a_msr5t2ph\",\"value\":111}]}" + "notes": "status 200: {\"data\":[{\"user_id\":\"3788dfbd-cd15-42c6-b452-c86a06f1d822\",\"metric\":\"steps_a_mssyktn2\",\"value\":111}]}" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"aae20fc0-9396-43c2-bd78-2df3e1594f54\",\"metric\":\"steps_b_msr5t2ph\",\"value\":222}]}" + "notes": "status 200: {\"data\":[{\"user_id\":\"2f9eb765-5224-4b11-b49f-418ff2696ae2\",\"metric\":\"steps_b_mssyktn2\",\"value\":222}]}" }, { "name": "non-service key is not granted service access", @@ -734,70 +845,11 @@ }, "docs": { "calls": [ - { - "source": "web_search", - "query": "npm @supabase/server withSupabase edge function package", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/blog/introducing-supabase-server", - "title": "Introducing @supabase/server" - }, - { - "url": "https://supabase.com/blog/edge-functions-node-npm", - "title": "Edge Functions: Node and native npm compatibility" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres) | Supabase Docs" - }, - { - "url": "https://supabase.com/docs/guides/functions/quickstart", - "title": "Getting Started with Edge Functions | Supabase Docs" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions | Supabase Docs" - }, - { - "url": "https://supabase.com/docs/guides/functions/wasm", - "title": "Using Wasm modules - Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/routing", - "title": "Handling Routing in Functions | Supabase Docs" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions | Supabase Docs" - } - ], - "resultChars": 2726 - }, - { - "source": "web_fetch", - "query": "Extract the full API documentation and code examples for the withSupabase function, including all auth modes (user, service, dual/either), how it reads the apikey/authorization headers, how ctx.supabase and ctx.supabaseAdmin are constructed, and any example of handling both a user JWT and a service-role caller in the same function.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/blog/introducing-supabase-server" - } - ], - "resultChars": 1789 - }, { "source": "search_docs", - "query": "{ searchDocs(query: \"@supabase/server withSupabase auth mode secret user dual authentication edge function\", limit: 10) { totalCount nodes { title href content } } }", + "query": "{ searchDocs(query: \"@supabase/server withSupabase edge function npm package\", limit: 10) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, { "url": "https://supabase.com/docs/guides/functions/auth", "title": "Securing Edge Functions" @@ -807,87 +859,39 @@ "title": "Which package to use" }, { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" + "url": "https://supabase.com/docs/guides/auth/server-side", + "title": "Server-Side Rendering" }, { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", + "title": "Serverless Drivers" }, { "url": "https://supabase.com/docs/guides/functions", "title": "Edge Functions" }, { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/functions/websockets", - "title": "Handling WebSockets" - }, - { - "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", - "title": "Getting Started with Edge Functions (Dashboard)" - } - ], - "resultChars": 78210 - }, - { - "source": "web_search", - "query": "\"@supabase/server\" withSupabase \"authMode\" OR \"ctx.authMode\" OR \"auth: ['user', 'secret']\" example", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions | Supabase Docs" - }, - { - "url": "https://supabase.com/blog/introducing-supabase-server", - "title": "Introducing @supabase/server" - }, - { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth | Supabase Docs" - }, - { - "url": "https://supabase.com/docs/reference/server/types-credentials", - "title": "Server API Reference | Supabase Docs" - }, - { - "url": "https://supabase.com/features/server-side-auth", - "title": "Server-side Auth | Supabase Features" - } - ], - "resultChars": 2668 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"verify_jwt config.toml functions combining auth modes secret user apikey header disable\", limit: 5) { totalCount nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/deploy", - "title": "Deploy to Production" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" }, { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips" + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", + "title": "Streaming Speech with ElevenLabs" }, { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" }, { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" + "url": "https://supabase.com/docs/guides/functions/quickstart-dashboard", + "title": "Getting Started with Edge Functions (Dashboard)" }, { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" } ], - "resultChars": 51704 + "resultChars": 108659 } ] }, @@ -925,7 +929,7 @@ { "name": "did not recommend read replicas for Realtime", "passed": true, - "judgeNotes": "The assistant correctly treats the issue as Supabase Realtime/Postgres Changes setup, adds the table to the supabase_realtime publication, discusses RLS and client subscription code, and does not recommend or imply read replicas are needed or useful for live updates." + "judgeNotes": "The answer correctly treats the task as Supabase Realtime/Postgres Changes setup by adding the messages table to the supabase_realtime publication and showing a postgres_changes client subscription. It does not recommend read replicas or confuse them with Realtime/logical replication." } ], "skills": { @@ -963,17 +967,17 @@ { "name": "identified video-thumbnails and the 546 resource-limit pattern", "passed": true, - "judgeNotes": "Identified `video-thumbnails` as the affected function and correctly recognized HTTP 546 resource/CPU limit failures rather than 500/503 server errors." + "judgeNotes": "The assistant correctly identified `video-thumbnails` as the affected function and explicitly recognized the failures as HTTP 546 responses due to a CPU/resource limit, not 500s/503s or boot/timeout errors." }, { "name": "attributed the 546s to CPU time exhaustion", "passed": true, - "judgeNotes": "The assistant explicitly attributes the 546 failures to CPU time exhaustion, citing the log shutdown reason CPUTime and cpu_time_used/cpu_time_limit of 2000ms." + "judgeNotes": "Assistant specifically attributes the 546 failures to CPU time exhaustion, citing shutdown reason CPUTime and cpu_time_used/cpu_time_limit at 2000ms, and explicitly rules out wall-clock, memory, and exceptions." }, { "name": "recommended reducing/offloading CPU work as the fix", "passed": true, - "judgeNotes": "The assistant clearly identifies the fixed CPU limit and recommends reducing CPU work per call, optimizing seeking/decoding, reducing input size, and moving heavy thumbnail generation to a background/external worker. It also explicitly rejects retries/scaling as fixes." + "judgeNotes": "Recommended reducing per-invocation CPU work and moving video thumbnail generation to a background/container worker, explicitly noting the CPU limit cannot be raised." } ], "skills": { @@ -1055,7 +1059,7 @@ { "name": "diagnosed secure default grants without weakening RLS", "passed": true, - "judgeNotes": "Diagnoses secure-by-default/missing Data API grants, distinguishes grants from RLS, preserves existing owner-scoped RLS, grants only SELECT/INSERT on public.journal_entries to authenticated, and keeps anon/public ungranted." + "judgeNotes": "The answer correctly identifies missing table-level grants for authenticated Data API access while preserving existing owner-scoped RLS, grants only SELECT/INSERT on public.journal_entries to authenticated, does not grant anon/public, and keeps RLS/security isolation intact. It distinguishes grants from RLS and verifies user isolation." } ], "skills": { @@ -1098,7 +1102,7 @@ { "name": "user A's update actually checks off their own task", "passed": true, - "notes": "saw: [{\"id\":\"ef42dd34-2cac-4bd1-9c47-bbf8019b7e0d\",\"is_done\":true}]" + "notes": "saw: [{\"id\":\"923a91f2-a3a3-456f-9937-344fd0f5220c\",\"is_done\":true}]" }, { "name": "user B cannot update user A's task", @@ -1107,7 +1111,7 @@ { "name": "diagnosed the missing USING clause and added it", "passed": true, - "judgeNotes": "The answer correctly identifies the missing UPDATE USING clause as the cause of silent zero-row updates, explains WITH CHECK vs USING, and fixes the authenticated task-owner policy with USING (user_id = auth.uid()) while preserving WITH CHECK (user_id = auth.uid()) and RLS." + "judgeNotes": "The assistant correctly identifies the missing USING clause on the UPDATE RLS policy as the cause of silent zero-row updates, explains the USING vs WITH CHECK behavior, and fixes the policy for authenticated users with USING (user_id = auth.uid()) while preserving WITH CHECK (user_id = auth.uid()). It does not disable RLS or broaden access." } ], "skills": { @@ -1118,7 +1122,7 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"row level security policy UPDATE USING WITH CHECK default behavior\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"row level security policy UPDATE USING WITH CHECK default\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -1126,23 +1130,23 @@ "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/auth/enterprise-sso/auth-sso-saml", - "title": "Single Sign-On with SAML 2.0 for Projects" + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" }, { "url": "https://supabase.com/docs/guides/troubleshooting/rls-simplified-BJTcS8", "title": "RLS Simplified" }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes", - "title": "Postgres Changes" + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" } ], - "resultChars": 125460 + "resultChars": 87537 } ] }, @@ -1174,7 +1178,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "The answer clearly says restart is the better first recovery step and that pause/restore is heavier and not the right first move. It distinguishes restart from pause/restore and offers concrete safe steps such as restarting and checking logs/advisors. It could be stronger by mentioning waiting, resource pressure/scaling, troubleshooting docs, and contacting Supabase support if unresolved." + "judgeNotes": "The answer clearly says pause/restore is not the right recovery step, distinguishes restart from pause/restore, recommends restarting the database/project and checking overload/scaling if it recurs, and offers logs/advisors investigation. It is grounded in Supabase troubleshooting and avoids conflating restore with restart." } ], "skills": { @@ -1182,7 +1186,36 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"project unhealthy restart pause dashboard\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "title": "Project Status reports unhealthy services" + }, + { + "url": "https://supabase.com/docs/guides/platform/delete-project", + "title": "Deleting Your Project" + }, + { + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" + }, + { + "url": "https://supabase.com/docs/guides/database/replication/pipelines-faq", + "title": "Pipelines FAQ" + }, + { + "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", + "title": "Restoring a downloaded backup locally" + } + ], + "resultChars": 30768 + } + ] }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", "promptSourcePath": "evals/resolve-reliability-001-unhealthy-project-recovery/PROMPT.md", @@ -1297,7 +1330,7 @@ { "name": "user A can replace their own avatar via upsert", "passed": true, - "notes": "saw: [{\"name\":\"019ff9df-4b74-73e9-a360-757cc38f38f8/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" + "notes": "saw: [{\"name\":\"01a0005d-5195-75fc-bbca-e07e5b1bd26c/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" }, { "name": "user B cannot overwrite user A's avatar", @@ -1306,7 +1339,7 @@ { "name": "added an owner-scoped UPDATE policy without weakening public reads", "passed": true, - "judgeNotes": "Diagnosed missing UPDATE RLS policy for upsert on storage.objects, kept public bucket/RLS intact, and added authenticated owner-scoped UPDATE policy with USING and WITH CHECK." + "judgeNotes": "The answer correctly diagnoses missing UPDATE RLS policy for upsert replacement, explains public bucket only covers reads, adds authenticated owner-scoped UPDATE policy with USING and WITH CHECK, and does not weaken public-read setup or disable RLS." } ], "skills": {