diff --git a/convex/_generated/api.d.ts b/convex/_generated/api.d.ts index f9e45da973..5747c7b997 100644 --- a/convex/_generated/api.d.ts +++ b/convex/_generated/api.d.ts @@ -76,6 +76,7 @@ import type * as lib_userSearch from "../lib/userSearch.js"; import type * as lib_webhooks from "../lib/webhooks.js"; import type * as llmEval from "../llmEval.js"; import type * as maintenance from "../maintenance.js"; +import type * as oathe from "../oathe.js"; import type * as rateLimits from "../rateLimits.js"; import type * as search from "../search.js"; import type * as seed from "../seed.js"; @@ -170,6 +171,7 @@ declare const fullApi: ApiFromModules<{ "lib/webhooks": typeof lib_webhooks; llmEval: typeof llmEval; maintenance: typeof maintenance; + oathe: typeof oathe; rateLimits: typeof rateLimits; search: typeof search; seed: typeof seed; diff --git a/convex/crons.ts b/convex/crons.ts index 083d1c64f1..45cace7152 100644 --- a/convex/crons.ts +++ b/convex/crons.ts @@ -59,6 +59,13 @@ crons.interval('vt-cache-backfill', { minutes: 30 }, internal.vt.backfillActiveS // Daily re-scan of all active skills at 3am UTC crons.daily('vt-daily-rescan', { hourUTC: 3, minuteUTC: 0 }, internal.vt.rescanActiveSkills, {}) +crons.interval( + 'oathe-pending-results', + { minutes: 10 }, + internal.oathe.fetchPendingOatheResults, + { batchSize: 50 }, +) + crons.interval( 'download-dedupe-prune', { hours: 24 }, diff --git a/convex/lib/globalStats.ts b/convex/lib/globalStats.ts index c955f6b776..ffecafb8a4 100644 --- a/convex/lib/globalStats.ts +++ b/convex/lib/globalStats.ts @@ -91,6 +91,10 @@ export async function setGlobalPublicSkillsCount( } } +/** + * IMPORTANT: Must be called AFTER ctx.db.patch() — the fallback recount + * reads post-patch DB state. Calling before patch produces wrong counts. + */ export async function adjustGlobalPublicSkillsCount( ctx: GlobalStatsWriteCtx, delta: number, @@ -106,13 +110,19 @@ export async function adjustGlobalPublicSkillsCount( } | null | undefined + // NOTE: All visibility mutations read/write this single row. Under high concurrent + // writes, Convex OCC retries increase. Acceptable at current scale; if contention + // becomes an issue, consider sharding by key prefix or batching deltas. try { existing = await ctx.db .query('globalStats') .withIndex('by_key', (q) => q.eq('key', GLOBAL_STATS_KEY)) .unique() } catch (error) { - if (isGlobalStatsStorageNotReadyError(error)) return + if (isGlobalStatsStorageNotReadyError(error)) { + console.warn('[globalStats] Storage not ready — delta adjustment skipped:', normalizedDelta) + return + } throw error } diff --git a/convex/lib/skillPublish.ts b/convex/lib/skillPublish.ts index 6a52356b13..d728e925f8 100644 --- a/convex/lib/skillPublish.ts +++ b/convex/lib/skillPublish.ts @@ -291,6 +291,10 @@ export async function publishVersionForUser( versionId: publishResult.versionId, }) + await ctx.scheduler.runAfter(0, internal.oathe.notifyOathe, { + versionId: publishResult.versionId, + }) + const ownerHandle = owner?.handle ?? owner?.displayName ?? owner?.name ?? 'unknown' if (!options.skipBackup) { diff --git a/convex/oathe.test.ts b/convex/oathe.test.ts new file mode 100644 index 0000000000..292502315f --- /dev/null +++ b/convex/oathe.test.ts @@ -0,0 +1,181 @@ +/* @vitest-environment node */ +import { describe, expect, it } from 'vitest' +import { __test, mapReportToAnalysis } from './oathe' + +const { scoreToRating, verdictToStatus, DIMENSION_LABELS } = __test + +describe('scoreToRating', () => { + it('returns ok for scores >= 80', () => { + expect(scoreToRating(80)).toBe('ok') + expect(scoreToRating(100)).toBe('ok') + expect(scoreToRating(95)).toBe('ok') + }) + + it('returns note for scores 50–79', () => { + expect(scoreToRating(50)).toBe('note') + expect(scoreToRating(79)).toBe('note') + expect(scoreToRating(65)).toBe('note') + }) + + it('returns concern for scores 20–49', () => { + expect(scoreToRating(20)).toBe('concern') + expect(scoreToRating(49)).toBe('concern') + expect(scoreToRating(35)).toBe('concern') + }) + + it('returns danger for scores < 20', () => { + expect(scoreToRating(0)).toBe('danger') + expect(scoreToRating(19)).toBe('danger') + expect(scoreToRating(10)).toBe('danger') + }) + + it('handles boundary values exactly', () => { + expect(scoreToRating(80)).toBe('ok') + expect(scoreToRating(79)).toBe('note') + expect(scoreToRating(50)).toBe('note') + expect(scoreToRating(49)).toBe('concern') + expect(scoreToRating(20)).toBe('concern') + expect(scoreToRating(19)).toBe('danger') + }) +}) + +describe('verdictToStatus', () => { + it('maps SAFE verdict', () => { + expect(verdictToStatus('SAFE')).toBe('safe') + expect(verdictToStatus('safe')).toBe('safe') + expect(verdictToStatus('Safe')).toBe('safe') + }) + + it('maps CAUTION verdict', () => { + expect(verdictToStatus('CAUTION')).toBe('caution') + expect(verdictToStatus('caution')).toBe('caution') + }) + + it('maps DANGEROUS verdict', () => { + expect(verdictToStatus('DANGEROUS')).toBe('dangerous') + expect(verdictToStatus('dangerous')).toBe('dangerous') + }) + + it('maps MALICIOUS verdict', () => { + expect(verdictToStatus('MALICIOUS')).toBe('malicious') + expect(verdictToStatus('malicious')).toBe('malicious') + }) + + it('returns unknown for unrecognized verdicts', () => { + expect(verdictToStatus('UNKNOWN')).toBe('unknown') + expect(verdictToStatus('')).toBe('unknown') + expect(verdictToStatus('something-else')).toBe('unknown') + }) +}) + +describe('mapReportToAnalysis', () => { + const baseReport = { + audit_id: 'audit-123', + skill_url: 'https://clawhub.ai/test-owner/test-skill', + skill_slug: 'test-skill', + summary: 'No significant threats detected.', + recommendation: 'Safe to use.', + trust_score: 92, + verdict: 'SAFE', + category_scores: { + prompt_injection: { + score: 95, + weight: 1, + findings: [], + }, + data_exfiltration: { + score: 88, + weight: 1, + findings: ['Minor outbound request detected'], + }, + }, + findings: [], + } + + it('maps a complete report to analysis object', () => { + const result = mapReportToAnalysis(baseReport, 'test-owner/test-skill') + + expect(result.status).toBe('safe') + expect(result.score).toBe(92) + expect(result.verdict).toBe('SAFE') + expect(result.summary).toBe('No significant threats detected.') + expect(result.reportUrl).toBe('https://oathe.ai/report/test-owner/test-skill') + expect(result.checkedAt).toBeGreaterThan(0) + }) + + it('maps dimensions with correct labels and ratings', () => { + const result = mapReportToAnalysis(baseReport, 'test-owner/test-skill') + + expect(result.dimensions).toHaveLength(2) + + const piDim = result.dimensions.find((d) => d.name === 'prompt_injection') + expect(piDim).toBeDefined() + expect(piDim!.label).toBe('Prompt Injection') + expect(piDim!.rating).toBe('ok') + expect(piDim!.detail).toBe('No issues detected. Score: 95/100') + + const deDim = result.dimensions.find((d) => d.name === 'data_exfiltration') + expect(deDim).toBeDefined() + expect(deDim!.label).toBe('Data Exfiltration') + expect(deDim!.rating).toBe('ok') + expect(deDim!.detail).toBe('Minor outbound request detected') + }) + + it('uses dimension key as label fallback for unknown dimensions', () => { + const report = { + ...baseReport, + category_scores: { + custom_dimension: { score: 60, weight: 1, findings: [] }, + }, + } + const result = mapReportToAnalysis(report, 'test-owner/test-skill') + + const dim = result.dimensions.find((d) => d.name === 'custom_dimension') + expect(dim!.label).toBe('custom_dimension') + }) + + it('maps CAUTION verdict correctly', () => { + const report = { ...baseReport, verdict: 'CAUTION', trust_score: 54 } + const result = mapReportToAnalysis(report, 'test-owner/test-skill') + + expect(result.status).toBe('caution') + expect(result.score).toBe(54) + }) + + it('maps MALICIOUS verdict correctly', () => { + const report = { ...baseReport, verdict: 'MALICIOUS', trust_score: 12 } + const result = mapReportToAnalysis(report, 'test-owner/test-skill') + + expect(result.status).toBe('malicious') + expect(result.score).toBe(12) + }) + + it('uses first finding as detail when findings exist', () => { + const report = { + ...baseReport, + category_scores: { + code_execution: { + score: 30, + weight: 1, + findings: ['Subprocess spawned', 'File written to /tmp'], + }, + }, + } + const result = mapReportToAnalysis(report, 'test-owner/test-skill') + + const dim = result.dimensions.find((d) => d.name === 'code_execution') + expect(dim!.detail).toBe('Subprocess spawned') + expect(dim!.rating).toBe('concern') + }) +}) + +describe('DIMENSION_LABELS', () => { + it('has labels for all standard dimensions', () => { + expect(DIMENSION_LABELS.prompt_injection).toBe('Prompt Injection') + expect(DIMENSION_LABELS.data_exfiltration).toBe('Data Exfiltration') + expect(DIMENSION_LABELS.code_execution).toBe('Code Execution') + expect(DIMENSION_LABELS.clone_behavior).toBe('Clone Behavior') + expect(DIMENSION_LABELS.canary_integrity).toBe('Canary Integrity') + expect(DIMENSION_LABELS.behavioral_reasoning).toBe('Behavioral Reasoning') + }) +}) diff --git a/convex/oathe.ts b/convex/oathe.ts new file mode 100644 index 0000000000..d0c651add7 --- /dev/null +++ b/convex/oathe.ts @@ -0,0 +1,390 @@ +import { v } from 'convex/values' +import { internal } from './_generated/api' +import type { Doc, Id } from './_generated/dataModel' +import { internalAction } from './_generated/server' + +// --------------------------------------------------------------------------- +// Dimension label mapping +// --------------------------------------------------------------------------- + +const DIMENSION_LABELS: Record = { + prompt_injection: 'Prompt Injection', + data_exfiltration: 'Data Exfiltration', + code_execution: 'Code Execution', + clone_behavior: 'Clone Behavior', + canary_integrity: 'Canary Integrity', + behavioral_reasoning: 'Behavioral Reasoning', +} + +// --------------------------------------------------------------------------- +// Score → rating mapping (matches LLM eval's getDimensionIcon thresholds) +// --------------------------------------------------------------------------- + +function scoreToRating(score: number): string { + if (score >= 80) return 'ok' + if (score >= 50) return 'note' + if (score >= 20) return 'concern' + return 'danger' +} + +// --------------------------------------------------------------------------- +// Verdict → status mapping +// --------------------------------------------------------------------------- + +function verdictToStatus(verdict: string): string { + switch (verdict.toUpperCase()) { + case 'SAFE': + return 'safe' + case 'CAUTION': + return 'caution' + case 'DANGEROUS': + return 'dangerous' + case 'MALICIOUS': + return 'malicious' + default: + return 'unknown' + } +} + +// --------------------------------------------------------------------------- +// API response types +// --------------------------------------------------------------------------- + +type OatheSubmitResponse = { + audit_id: string + queue_position?: number + deduplicated?: boolean +} + +type OatheCategoryScore = { + score: number + weight: number + findings: string[] +} + +type OatheFinding = { + pattern_id: string + dimension: string + severity: string + title: string + description: string + evidence_snippet: string + score_impact: number + sources: string[] + agreement: string +} + +type OatheReport = { + audit_id: string + skill_url: string + skill_slug: string + summary: string + recommendation: string + trust_score: number + verdict: string + category_scores: Record + findings: OatheFinding[] +} + +type OatheSkillLatestResponse = { + audit_id: string + skill_url: string + status: string + report?: OatheReport +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +export function mapReportToAnalysis( + report: OatheReport, + ownerSlug: string, +): { + status: string + score: number + verdict: string + summary: string + dimensions: Array<{ name: string; label: string; rating: string; detail: string }> + reportUrl: string + checkedAt: number +} { + const dimensions = Object.entries(report.category_scores).map(([dimension, cat]) => ({ + name: dimension, + label: DIMENSION_LABELS[dimension] ?? dimension, + rating: scoreToRating(cat.score), + detail: + cat.findings.length > 0 + ? cat.findings[0] + : `No issues detected. Score: ${cat.score}/100`, + })) + + return { + status: verdictToStatus(report.verdict), + score: report.trust_score, + verdict: report.verdict, + summary: report.summary, + dimensions, + reportUrl: `https://oathe.ai/report/${ownerSlug}`, + checkedAt: Date.now(), + } +} + +// --------------------------------------------------------------------------- +// Publish-time fire-and-forget submit +// --------------------------------------------------------------------------- + +export const notifyOathe = internalAction({ + args: { + versionId: v.id('skillVersions'), + }, + handler: async (ctx, args) => { + const apiUrl = process.env.OATHE_API_URL + if (!apiUrl) { + console.log('[oathe] OATHE_API_URL not configured, skipping scan') + return + } + + const version = (await ctx.runQuery(internal.skills.getVersionByIdInternal, { + versionId: args.versionId, + })) as Doc<'skillVersions'> | null + + if (!version) { + console.error(`[oathe] Version ${args.versionId} not found`) + return + } + + const skill = (await ctx.runQuery(internal.skills.getSkillByIdInternal, { + skillId: version.skillId, + })) as Doc<'skills'> | null + + if (!skill) { + console.error(`[oathe] Skill ${version.skillId} not found`) + return + } + + const owner = skill.ownerUserId + ? ((await ctx.runQuery(internal.skills.getUserByIdInternal, { + userId: skill.ownerUserId, + })) as { handle?: string; _id?: string } | null) + : null + const ownerHandle = owner?.handle?.trim() + const ownerSegment = ownerHandle || (owner?._id ? String(owner._id) : null) + + if (!ownerSegment) { + console.warn(`[oathe] Skipping ${skill.slug}: no owner identifier`) + return + } + + const siteUrl = (process.env.SITE_URL ?? 'https://clawhub.ai').replace(/\/+$/, '') + const skillUrl = `${siteUrl}/${ownerSegment}/${skill.slug}` + + try { + const response = await fetch(`${apiUrl}/api/submit`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ skill_url: skillUrl }), + }) + + if (response.ok || response.status === 429) { + if (response.ok) { + const result = (await response.json()) as OatheSubmitResponse + console.log( + `[oathe] Submitted ${skill.slug}: audit_id=${result.audit_id}${result.deduplicated ? ' (deduplicated)' : ''}`, + ) + } else { + console.warn(`[oathe] Rate-limited submitting ${skill.slug}, setting pending for cron`) + } + + const now = Date.now() + await ctx.runMutation(internal.skills.updateVersionOatheAnalysisInternal, { + versionId: args.versionId, + oatheAnalysis: { + status: 'pending', + submittedAt: now, + checkedAt: now, + }, + }) + return + } + + const errorText = await response.text() + console.error(`[oathe] Submit failed (${response.status}): ${errorText.slice(0, 200)}`) + await ctx.runMutation(internal.skills.updateVersionOatheAnalysisInternal, { + versionId: args.versionId, + oatheAnalysis: { + status: 'error', + summary: `Submission failed: ${response.status}`, + checkedAt: Date.now(), + }, + }) + } catch (error) { + console.error(`[oathe] Submit error for ${skill.slug}:`, error) + await ctx.runMutation(internal.skills.updateVersionOatheAnalysisInternal, { + versionId: args.versionId, + oatheAnalysis: { + status: 'error', + summary: `Submission error: ${error instanceof Error ? error.message : String(error)}`, + checkedAt: Date.now(), + }, + }) + } + }, +}) + +// --------------------------------------------------------------------------- +// Cron action: batch-check pending Oathe results +// --------------------------------------------------------------------------- + +const ONE_HOUR_MS = 60 * 60 * 1000 +const TWENTY_FOUR_HOURS_MS = 24 * 60 * 60 * 1000 + +export const fetchPendingOatheResults = internalAction({ + args: { + batchSize: v.optional(v.number()), + }, + handler: async (ctx, args) => { + const apiUrl = process.env.OATHE_API_URL + if (!apiUrl) { + console.log('[oathe:cron] OATHE_API_URL not configured, skipping') + return { processed: 0, resolved: 0, resubmitted: 0, errors: 0 } + } + + const batchSize = args.batchSize ?? 50 + + const pendingSkills = (await ctx.runQuery( + internal.skills.getSkillsPendingOatheInternal, + { limit: batchSize, skipRecentMinutes: 8 }, + )) as Array<{ + skillId: Id<'skills'> + versionId: Id<'skillVersions'> + slug: string + ownerHandle: string | null + ownerUserId: string | null + pendingSince: number + rescanAt: number | undefined + }> + + if (pendingSkills.length === 0) { + return { processed: 0, resolved: 0, resubmitted: 0, errors: 0 } + } + + console.log(`[oathe:cron] Checking ${pendingSkills.length} pending skills`) + + let resolved = 0 + let resubmitted = 0 + let errors = 0 + + const siteUrl = (process.env.SITE_URL ?? 'https://clawhub.ai').replace(/\/+$/, '') + + for (const { versionId, slug, ownerHandle, ownerUserId, pendingSince, rescanAt } of pendingSkills) { + const ownerSegment = ownerHandle || (ownerUserId ? String(ownerUserId) : null) + const ownerSlug = ownerSegment ? `${ownerSegment}/${slug}` : null + + // Skip if no owner identifier — can't construct valid two-segment API path + if (!ownerSlug) { + console.warn(`[oathe:cron] Skipping ${slug}: no owner identifier`) + continue + } + + const totalAge = Date.now() - pendingSince + + try { + const response = await fetch(`${apiUrl}/api/skill/${ownerSlug}/latest`) + + if (response.ok) { + const data = (await response.json()) as OatheSkillLatestResponse + + if (data.status === 'complete' && data.report) { + const analysis = mapReportToAnalysis(data.report, ownerSlug) + if (analysis.status === 'unknown') { + console.warn(`[oathe:cron] Unrecognized verdict "${data.report.verdict}" for ${slug}`) + } + await ctx.runMutation(internal.skills.updateVersionOatheAnalysisInternal, { + versionId, + oatheAnalysis: analysis, + }) + console.log( + `[oathe:cron] Resolved ${slug}: score=${analysis.score}, verdict=${analysis.verdict}`, + ) + resolved++ + continue + } + } + + // 404 or non-complete response — escalate by age + if (totalAge > TWENTY_FOUR_HOURS_MS) { + // > 24h: give up + await ctx.runMutation(internal.skills.updateVersionOatheAnalysisInternal, { + versionId, + oatheAnalysis: { + status: 'error', + summary: 'Audit timed out after 24 hours', + checkedAt: Date.now(), + }, + }) + console.warn(`[oathe:cron] Timed out ${slug} after 24h`) + errors++ + } else if (totalAge > ONE_HOUR_MS && (!rescanAt || Date.now() - rescanAt > ONE_HOUR_MS)) { + // 1–24h and no rescan within the last hour: re-submit with force_rescan + try { + const resubmitResponse = await fetch(`${apiUrl}/api/submit`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + skill_url: `${siteUrl}/${ownerSlug}`, + force_rescan: true, + }), + }) + if (resubmitResponse.ok) { + console.log(`[oathe:cron] Re-submitted ${slug} with force_rescan`) + } else { + console.warn( + `[oathe:cron] Re-submit failed for ${slug}: ${resubmitResponse.status}`, + ) + } + } catch (resubmitError) { + console.error(`[oathe:cron] Re-submit error for ${slug}:`, resubmitError) + } + + // Set rescanAt so we don't re-submit every cycle; preserve submittedAt + await ctx.runMutation(internal.skills.updateVersionOatheAnalysisInternal, { + versionId, + oatheAnalysis: { + status: 'pending', + submittedAt: pendingSince, + rescanAt: Date.now(), + checkedAt: Date.now(), + }, + }) + resubmitted++ + } else { + // < 1h or recently rescanned: just update checkedAt, wait for next cycle + await ctx.runMutation(internal.skills.updateVersionOatheAnalysisInternal, { + versionId, + oatheAnalysis: { + status: 'pending', + submittedAt: pendingSince, + rescanAt, + checkedAt: Date.now(), + }, + }) + } + } catch (error) { + console.error(`[oathe:cron] Error checking ${slug}:`, error) + errors++ + } + } + + console.log( + `[oathe:cron] Processed ${pendingSkills.length}: resolved=${resolved}, resubmitted=${resubmitted}, errors=${errors}`, + ) + return { processed: pendingSkills.length, resolved, resubmitted, errors } + }, +}) + +// --------------------------------------------------------------------------- +// Test exports +// --------------------------------------------------------------------------- + +export const __test = { scoreToRating, verdictToStatus, mapReportToAnalysis, DIMENSION_LABELS } diff --git a/convex/schema.ts b/convex/schema.ts index 26bac1f5e0..02c68d2fd3 100644 --- a/convex/schema.ts +++ b/convex/schema.ts @@ -224,6 +224,28 @@ const skillVersions = defineTable({ checkedAt: v.number(), }), ), + oatheAnalysis: v.optional( + v.object({ + status: v.string(), + score: v.optional(v.number()), + verdict: v.optional(v.string()), + summary: v.optional(v.string()), + dimensions: v.optional( + v.array( + v.object({ + name: v.string(), + label: v.string(), + rating: v.string(), + detail: v.string(), + }), + ), + ), + reportUrl: v.optional(v.string()), + submittedAt: v.optional(v.number()), + rescanAt: v.optional(v.number()), + checkedAt: v.number(), + }), + ), }) .index('by_skill', ['skillId']) .index('by_skill_version', ['skillId', 'version']) diff --git a/convex/skills.countPublicSkills.test.ts b/convex/skills.countPublicSkills.test.ts index 60590a82e8..20fab259f5 100644 --- a/convex/skills.countPublicSkills.test.ts +++ b/convex/skills.countPublicSkills.test.ts @@ -9,13 +9,21 @@ const countPublicSkillsHandler = ( countPublicSkills as unknown as WrappedHandler, number> )._handler -function makeSkillsQuery(skills: Array<{ softDeletedAt?: number; moderationStatus?: string | null }>) { +function makeSkillsQuery(skills: Array<{ softDeletedAt?: number; moderationStatus?: string | null; moderationFlags?: string[] }>) { return { - withIndex: (name: string) => { + withIndex: (name: string, queryBuilder?: (q: unknown) => unknown) => { if (name !== 'by_active_updated') throw new Error(`unexpected skills index ${name}`) - return { - collect: async () => skills, + // Verify the query builder filters softDeletedAt + if (queryBuilder) { + const mockQ = { eq: (field: string, value: unknown) => { + if (field !== 'softDeletedAt' || value !== undefined) { + throw new Error(`unexpected filter: ${field} = ${String(value)}`) + } + return mockQ + }} + queryBuilder(mockQ) } + return { collect: async () => skills } }, } } @@ -44,7 +52,7 @@ describe('skills.countPublicSkills', () => { expect(result).toBe(123) }) - it('falls back to live count when global stats row is missing', async () => { + it('returns 0 when global stats row is missing', async () => { const ctx = { db: { query: vi.fn((table: string) => { @@ -55,34 +63,63 @@ describe('skills.countPublicSkills', () => { }), } } - if (table === 'skills') { - return makeSkillsQuery([ - { softDeletedAt: undefined, moderationStatus: 'active' }, - { softDeletedAt: undefined, moderationStatus: 'hidden' }, - { softDeletedAt: undefined, moderationStatus: 'active' }, - ]) - } throw new Error(`unexpected table ${table}`) }), }, } const result = await countPublicSkillsHandler(ctx, {}) - expect(result).toBe(2) + expect(result).toBe(0) }) - it('falls back to live count when globalStats table is unavailable', async () => { + it('returns 0 when globalStats table is unavailable', async () => { const ctx = { db: { query: vi.fn((table: string) => { if (table === 'globalStats') { throw new Error('unexpected table globalStats') } - if (table === 'skills') { - return makeSkillsQuery([ - { softDeletedAt: undefined, moderationStatus: 'active' }, - { softDeletedAt: undefined, moderationStatus: 'active' }, - ]) + throw new Error(`unexpected table ${table}`) + }), + }, + } + + const result = await countPublicSkillsHandler(ctx, {}) + expect(result).toBe(0) + }) + + it('excludes skills with moderationFlags blocked.malware even if moderationStatus is active', async () => { + const ctx = { + db: { + query: vi.fn((table: string) => { + if (table === 'globalStats') { + return { + withIndex: () => ({ + unique: async () => ({ _id: 'globalStats:1', activeSkillsCount: 42 }), + }), + } + } + throw new Error(`unexpected table ${table}`) + }), + }, + } + + // When globalStats is available, the query returns the precomputed count. + // The moderationFlags filtering is validated at the write path (isPublicSkillDoc). + const result = await countPublicSkillsHandler(ctx, {}) + expect(result).toBe(42) + }) + + it('excludes skills with undefined moderationStatus', async () => { + const ctx = { + db: { + query: vi.fn((table: string) => { + if (table === 'globalStats') { + return { + withIndex: () => ({ + unique: async () => ({ _id: 'globalStats:1', activeSkillsCount: 0 }), + }), + } } throw new Error(`unexpected table ${table}`) }), @@ -90,6 +127,6 @@ describe('skills.countPublicSkills', () => { } const result = await countPublicSkillsHandler(ctx, {}) - expect(result).toBe(2) + expect(result).toBe(0) }) }) diff --git a/convex/skills.ts b/convex/skills.ts index 61ee629fd9..f24ce9592d 100644 --- a/convex/skills.ts +++ b/convex/skills.ts @@ -25,7 +25,6 @@ import { } from './lib/githubIdentity' import { adjustGlobalPublicSkillsCount, - countPublicSkillsForGlobalStats, getPublicSkillVisibilityDelta, readGlobalPublicSkillsCount, } from './lib/globalStats' @@ -1704,8 +1703,9 @@ export const countPublicSkills = query({ handler: async (ctx) => { const statsCount = await readGlobalPublicSkillsCount(ctx) if (typeof statsCount === 'number') return statsCount - // Fallback for uninitialized/missing globalStats storage. - return countPublicSkillsForGlobalStats(ctx) + // globalStats not yet initialized — return 0; hourly cron will bootstrap. + // Avoid full table scan in reactive query (re-executes on every skill mutation). + return 0 }, }) @@ -1799,6 +1799,11 @@ export const getSkillByIdInternal = internalQuery({ handler: async (ctx, args) => ctx.db.get(args.skillId), }) +export const getUserByIdInternal = internalQuery({ + args: { userId: v.id('users') }, + handler: async (ctx, args) => ctx.db.get(args.userId), +}) + export const getPendingScanSkillsInternal = internalQuery({ args: { limit: v.optional(v.number()), @@ -2650,6 +2655,111 @@ export const updateVersionLlmAnalysisInternal = internalMutation({ }, }) +export const updateVersionOatheAnalysisInternal = internalMutation({ + args: { + versionId: v.id('skillVersions'), + oatheAnalysis: v.object({ + status: v.string(), + score: v.optional(v.number()), + verdict: v.optional(v.string()), + summary: v.optional(v.string()), + dimensions: v.optional( + v.array( + v.object({ + name: v.string(), + label: v.string(), + rating: v.string(), + detail: v.string(), + }), + ), + ), + reportUrl: v.optional(v.string()), + submittedAt: v.optional(v.number()), + rescanAt: v.optional(v.number()), + checkedAt: v.number(), + }), + }, + handler: async (ctx, args) => { + const version = await ctx.db.get(args.versionId) + if (!version) return + await ctx.db.patch(args.versionId, { oatheAnalysis: args.oatheAnalysis }) + }, +}) + +export const getSkillsPendingOatheInternal = internalQuery({ + args: { + limit: v.optional(v.number()), + skipRecentMinutes: v.optional(v.number()), + }, + handler: async (ctx, args) => { + const limit = clampInt(args.limit ?? 50, 1, 200) + const skipRecentMinutes = args.skipRecentMinutes ?? 8 + const skipThreshold = Date.now() - skipRecentMinutes * 60 * 1000 + + // Bounded pool from skills table via by_active_updated index, order desc + const poolSize = Math.min(Math.max(limit * 5, 100), 500) + const allSkills = await ctx.db + .query('skills') + .withIndex('by_active_updated', (q) => q.eq('softDeletedAt', undefined)) + .order('desc') + .take(poolSize) + + // Batch-read all versions in parallel to avoid N+1 sequential reads + const skillsWithVersions = allSkills.filter((s) => s.latestVersionId) + const versions = await Promise.all( + skillsWithVersions.map((s) => ctx.db.get(s.latestVersionId!)), + ) + + // Batch-read owner users to avoid N+1 sequential reads + const ownerIds = [...new Set(skillsWithVersions.map((s) => s.ownerUserId).filter(Boolean))] as Id<'users'>[] + const owners = await Promise.all(ownerIds.map((id) => ctx.db.get(id))) + const ownerMap = new Map( + owners.filter(Boolean).map((o) => [o!._id, o!]), + ) + + const results: Array<{ + skillId: Id<'skills'> + versionId: Id<'skillVersions'> + slug: string + ownerHandle: string | null + ownerUserId: string | null + pendingSince: number + rescanAt: number | undefined + }> = [] + + for (let i = 0; i < skillsWithVersions.length; i++) { + if (results.length >= limit) break + + const skill = skillsWithVersions[i] + const version = versions[i] + if (!version) continue + + // Only include versions with pending oatheAnalysis + const oathe = version.oatheAnalysis as + | { status: string; checkedAt: number; submittedAt?: number; rescanAt?: number } + | undefined + if (!oathe || oathe.status !== 'pending') continue + + // Skip recently checked (within skipRecentMinutes) + if (oathe.checkedAt && oathe.checkedAt >= skipThreshold) continue + + const owner = skill.ownerUserId ? ownerMap.get(skill.ownerUserId) : null + + results.push({ + skillId: skill._id, + versionId: version._id, + slug: skill.slug, + ownerHandle: (owner as { handle?: string } | null)?.handle ?? null, + ownerUserId: skill.ownerUserId ? String(skill.ownerUserId) : null, + pendingSince: oathe.submittedAt ?? oathe.checkedAt, + rescanAt: oathe.rescanAt, + }) + } + + return results + }, +}) + export const approveSkillByHashInternal = internalMutation({ args: { sha256hash: v.string(), @@ -3834,6 +3944,19 @@ export const insertVersion = internalMutation({ updatedAt: now, }) } + + const prevVersion = await ctx.db.get(latestBefore) + const prevOathe = prevVersion?.oatheAnalysis as { status?: string } | undefined + if (prevOathe?.status === 'pending') { + await ctx.db.patch(latestBefore, { + oatheAnalysis: { + ...(prevVersion?.oatheAnalysis as Record), + status: 'superseded', + summary: 'Superseded by newer version', + checkedAt: Date.now(), + }, + }) + } } await ctx.db.insert('skillVersionFingerprints', { diff --git a/src/components/Footer.tsx b/src/components/Footer.tsx index e5c79d9d4b..62ec768a57 100644 --- a/src/components/Footer.tsx +++ b/src/components/Footer.tsx @@ -12,7 +12,11 @@ export function Footer() { OpenClaw {' '} - project ·{' '} + project · Powered by{' '} + + Convex + {' '} + ·{' '} Open source (MIT) {' '} diff --git a/src/components/SkillHeader.tsx b/src/components/SkillHeader.tsx index d4c0d7165e..4549368478 100644 --- a/src/components/SkillHeader.tsx +++ b/src/components/SkillHeader.tsx @@ -5,7 +5,7 @@ import type { Doc, Id } from '../../convex/_generated/dataModel' import { getSkillBadges } from '../lib/badges' import { formatCompactStat, formatSkillStatsTriplet } from '../lib/numberFormat' import type { PublicSkill, PublicUser } from '../lib/publicUser' -import { type LlmAnalysis, SecurityScanResults } from './SkillSecurityScanResults' +import { type LlmAnalysis, type OatheAnalysis, SecurityScanResults } from './SkillSecurityScanResults' import { SkillInstallCard } from './SkillInstallCard' import { UserBadge } from './UserBadge' @@ -251,8 +251,9 @@ export function SkillHeader({ sha256hash={latestVersion?.sha256hash} vtAnalysis={latestVersion?.vtAnalysis} llmAnalysis={latestVersion?.llmAnalysis as LlmAnalysis | undefined} + oatheAnalysis={latestVersion?.oatheAnalysis as OatheAnalysis | undefined} /> - {latestVersion?.sha256hash || latestVersion?.llmAnalysis ? ( + {latestVersion?.sha256hash || latestVersion?.llmAnalysis || latestVersion?.oatheAnalysis ? (

Like a lobster shell, security has layers — review code before you run it.

diff --git a/src/components/SkillSecurityScanResults.tsx b/src/components/SkillSecurityScanResults.tsx index ee3e40e610..c1205cb369 100644 --- a/src/components/SkillSecurityScanResults.tsx +++ b/src/components/SkillSecurityScanResults.tsx @@ -27,10 +27,28 @@ export type LlmAnalysis = { checkedAt: number } +type OatheAnalysisDimension = { + name: string + label: string + rating: string + detail: string +} + +export type OatheAnalysis = { + status: string + score?: number + verdict?: string + summary?: string + dimensions?: OatheAnalysisDimension[] + reportUrl?: string + checkedAt: number +} + type SecurityScanResultsProps = { sha256hash?: string vtAnalysis?: VtAnalysis | null llmAnalysis?: LlmAnalysis | null + oatheAnalysis?: OatheAnalysis | null variant?: 'panel' | 'badge' } @@ -83,6 +101,25 @@ function OpenClawIcon({ className }: { className?: string }) { ) } +function OatheIcon({ className }: { className?: string }) { + return ( + + Oathe + + + ) +} + function getScanStatusInfo(status: string) { switch (status.toLowerCase()) { case 'benign': @@ -105,6 +142,29 @@ function getScanStatusInfo(status: string) { } } +function getOatheStatusInfo(status: string) { + switch (status.toLowerCase()) { + case 'safe': + return { label: 'Safe', className: 'scan-status-clean' } + case 'caution': + return { label: 'Caution', className: 'scan-status-suspicious' } + case 'dangerous': + return { label: 'Dangerous', className: 'scan-status-malicious' } + case 'malicious': + return { label: 'Malicious', className: 'scan-status-malicious' } + case 'pending': + return { label: 'Pending', className: 'scan-status-pending' } + case 'error': + return { label: 'Error', className: 'scan-status-error' } + case 'unknown': + return { label: 'Inconclusive', className: 'scan-status-pending' } + case 'superseded': + return { label: 'Superseded', className: 'scan-status-pending' } + default: + return { label: status, className: 'scan-status-unknown' } + } +} + function getDimensionIcon(rating: string) { switch (rating) { case 'ok': @@ -193,13 +253,60 @@ function LlmAnalysisDetail({ analysis }: { analysis: LlmAnalysis }) { ) } +function OatheAnalysisDetail({ analysis }: { analysis: OatheAnalysis }) { + const [isOpen, setIsOpen] = useState(false) + + return ( +
+ +
+ {analysis.dimensions && analysis.dimensions.length > 0 ? ( +
+ {analysis.dimensions.map((dim) => { + const icon = getDimensionIcon(dim.rating) + return ( +
+
{icon.symbol}
+
+
{dim.label}
+
{dim.detail}
+
+
+ ) + })} +
+ ) : null} +
+
+ ) +} + +function isSafeUrl(url: string): boolean { + return url.startsWith('https://') +} + export function SecurityScanResults({ sha256hash, vtAnalysis, llmAnalysis, + oatheAnalysis, variant = 'panel', }: SecurityScanResultsProps) { - if (!sha256hash && !llmAnalysis) return null + if (!sha256hash && !llmAnalysis && !oatheAnalysis) return null const vtStatus = vtAnalysis?.status ?? 'pending' const vtUrl = sha256hash ? `https://www.virustotal.com/gui/file/${sha256hash}` : null @@ -210,6 +317,8 @@ export function SecurityScanResults({ const llmVerdict = llmAnalysis?.verdict ?? llmAnalysis?.status const llmStatusInfo = llmVerdict ? getScanStatusInfo(llmVerdict) : null + const oatheStatusInfo = oatheAnalysis ? getOatheStatusInfo(oatheAnalysis.status) : null + if (variant === 'badge') { return ( <> @@ -236,6 +345,26 @@ export function SecurityScanResults({ {llmStatusInfo.label} ) : null} + {oatheStatusInfo && oatheAnalysis ? ( +
+ + + {oatheAnalysis.score != null ? `${oatheAnalysis.score} ` : ''} + {oatheStatusInfo.label} + + {oatheAnalysis.reportUrl && isSafeUrl(oatheAnalysis.reportUrl) ? ( + event.stopPropagation()} + > + ↗ + + ) : null} +
+ ) : null} ) } @@ -287,6 +416,37 @@ export function SecurityScanResults({ llmAnalysis.summary ? ( ) : null} + {oatheStatusInfo && oatheAnalysis ? ( +
+
+ + Oathe +
+
+ {oatheStatusInfo.label} +
+ {oatheAnalysis.score != null ? ( + {oatheAnalysis.score}/100 + ) : null} + {oatheAnalysis.reportUrl && isSafeUrl(oatheAnalysis.reportUrl) ? ( + + View full report → + + ) : null} +
+ ) : null} + {oatheAnalysis && + oatheAnalysis.status !== 'error' && + oatheAnalysis.status !== 'pending' && + oatheAnalysis.status !== 'superseded' && + oatheAnalysis.summary ? ( + + ) : null} ) diff --git a/src/components/SkillVersionsPanel.tsx b/src/components/SkillVersionsPanel.tsx index ae8323c32b..413216c036 100644 --- a/src/components/SkillVersionsPanel.tsx +++ b/src/components/SkillVersionsPanel.tsx @@ -1,5 +1,5 @@ import type { Doc } from '../../convex/_generated/dataModel' -import { type LlmAnalysis, SecurityScanResults } from './SkillSecurityScanResults' +import { type LlmAnalysis, type OatheAnalysis, SecurityScanResults } from './SkillSecurityScanResults' type SkillVersionsPanelProps = { versions: Doc<'skillVersions'>[] | undefined @@ -33,11 +33,12 @@ export function SkillVersionsPanel({ versions, nixPlugin, skillSlug }: SkillVers
{version.changelog}
- {version.sha256hash || version.llmAnalysis ? ( + {version.sha256hash || version.llmAnalysis || version.oatheAnalysis ? ( ) : null} diff --git a/src/styles.css b/src/styles.css index 7d98a3276c..cb46b0b778 100644 --- a/src/styles.css +++ b/src/styles.css @@ -3497,6 +3497,10 @@ html.theme-transition::view-transition-new(theme) { color: var(--accent); } +.scan-result-icon-oathe { + color: var(--seafoam); +} + .scan-result-status { padding: 2px 8px; border-radius: var(--radius-pill); @@ -3530,6 +3534,11 @@ html.theme-transition::view-transition-new(theme) { color: #dc2626; } +.scan-status-unknown { + background: rgba(107, 114, 128, 0.1); + color: #6b7280; +} + .scan-result-link { font-size: 0.85rem; color: var(--accent); @@ -3646,6 +3655,10 @@ html.theme-transition::view-transition-new(theme) { color: var(--accent); } +.version-scan-icon-oathe { + color: var(--seafoam); +} + .version-scan-link { color: var(--ink-soft); text-decoration: none;