Simulation Environment

Author: csansoon

apps/gateway/src/services/cloudwatchMetrics.ts

@@ -24,7 +24,11 @@ class CloudWatchMetricsService {
   }
 
   private isConfigured(): boolean {
-    return Boolean(env.AWS_ACCESS_KEY && env.AWS_ACCESS_SECRET)
+    return Boolean(
+      env.NODE_ENV === 'production' &&
+        env.AWS_ACCESS_KEY &&
+        env.AWS_ACCESS_SECRET,
+    )
   }
 
   incrementInflightRequests(): void {

apps/web/src/components/ChatWrapper/Message/Content/ToolCall/_components/ToolCard/Content.tsx

@@ -11,13 +11,16 @@ import { Text } from '@latitude-data/web-ui/atoms/Text'
 import { stringifyUnknown } from '@latitude-data/web-ui/textUtils'
 import { cn } from '@latitude-data/web-ui/utils'
 import { ReactNode } from 'react'
+import { SimulationTag } from './SimulationTag'
 
 export function ToolCardContentWrapper({
   badge,
+  simulated,
   className,
   children,
 }: {
   badge?: string
+  simulated?: boolean
   className?: string
   children: ReactNode
 }) {
@@ -28,9 +31,10 @@ export function ToolCardContentWrapper({
         className,
       )}
     >
-      {badge && (
-        <div className='flex flex-row'>
-          <Badge variant='outline'>{badge}</Badge>
+      {(badge || simulated) && (
+        <div className='flex flex-row gap-2'>
+          {badge && <Badge variant='outline'>{badge}</Badge>}
+          {simulated && <SimulationTag />}
         </div>
       )}
       <MaxHeightWindow>{children}</MaxHeightWindow>
@@ -54,11 +58,13 @@ export function ToolCardInput({
 
 export function ToolCardOutput({
   toolResponse,
+  simulated,
 }: {
   toolResponse: ToolContent | undefined
+  simulated?: boolean
 }) {
   return (
-    <ToolCardContentWrapper badge='Output'>
+    <ToolCardContentWrapper badge='Output' simulated={simulated}>
       {toolResponse ? (
         toolResponse.isError ? (
           <div className='w-full pt-3 items-center'>

apps/web/src/components/ChatWrapper/Message/Content/ToolCall/_components/ToolCard/Header.tsx

@@ -2,6 +2,7 @@ import { Icon, IconName } from '@latitude-data/web-ui/atoms/Icons'
 import { TextColor } from '@latitude-data/web-ui/tokens'
 import { cn } from '@latitude-data/web-ui/utils'
 import { ReactNode } from 'react'
+import { SimulationTag } from './SimulationTag'
 
 const statusIconColor = (
   status: 'pending' | 'success' | 'error',
@@ -35,12 +36,14 @@ export function ToolCardHeader({
   status,
   isOpen,
   onToggle,
+  simulated,
 }: {
   icon: ReactNode
   label: ReactNode
   status?: 'pending' | 'success' | 'error' | undefined
   isOpen: boolean
   onToggle?: () => void
+  simulated?: boolean
 }) {
   return (
     <div
@@ -71,6 +74,7 @@ export function ToolCardHeader({
         )}
       </div>
       <div className='flex flex-col min-w-0 flex-1'>{label}</div>
+      {simulated && <SimulationTag />}
       {onToggle && (
         <Icon
           name={isOpen ? 'chevronUp' : 'chevronDown'}

apps/web/src/components/ChatWrapper/Message/Content/ToolCall/_components/ToolCard/SimulationTag.tsx

@@ -0,0 +1,23 @@
+import { Badge } from '@latitude-data/web-ui/atoms/Badge'
+import { Icon } from '@latitude-data/web-ui/atoms/Icons'
+import { Text } from '@latitude-data/web-ui/atoms/Text'
+import { Tooltip } from '@latitude-data/web-ui/atoms/Tooltip'
+
+export function SimulationTag() {
+  return (
+    <Tooltip
+      asChild
+      trigger={
+        <Badge variant='warningMuted'>
+          <div className='flex flex-row items-center gap-1'>
+            <Icon name='cat' color='warningMutedForeground' />
+            <Text.H6B color='warningMutedForeground'>Simulated</Text.H6B>
+          </div>
+        </Badge>
+      }
+    >
+      This tool has been simulated. The real tool has not been executed.
+      Instead, an LLM has generated the results.
+    </Tooltip>
+  )
+}

apps/web/src/components/ChatWrapper/Message/Content/ToolCall/_components/ToolCard/index.tsx

@@ -95,9 +95,15 @@ export function ToolCard({
         status={status}
         isOpen={isOpen}
         onToggle={() => setIsOpen(!isOpen)}
+        simulated={toolRequest._sourceData?.simulated}
       />
       {isOpen && <ToolCardInput toolRequest={toolRequest} />}
-      {isOpen && <ToolCardOutput toolResponse={toolResponse} />}
+      {isOpen && (
+        <ToolCardOutput
+          toolResponse={toolResponse}
+          simulated={toolRequest._sourceData?.simulated}
+        />
+      )}
     </ToolCardWrapper>
   )
 }

packages/constants/src/index.ts

@@ -62,6 +62,11 @@ export type ExperimentScores = {
   [evaluationUuid: string]: ExperimentEvaluationScore
 }
 
+export type SimulationSettings = {
+  simulationDescription?: string // A prompt used to guide and generate the simulation result
+  simulateToolResponses?: boolean
+}
+
 // TODO: Remove these
 export * from './ai'
 export * from './config'

packages/constants/src/toolSources.ts

@@ -10,6 +10,7 @@ export enum ToolSource {
 }
 
 type BaseToolSourceData<T extends ToolSource> = {
+  simulated?: boolean
   source: T
 }
 

packages/core/src/jobs/job-definitions/documents/runDocumentAtCommitWithAutoToolResponses/getDataForInitialRequest.ts

@@ -7,7 +7,7 @@ import * as factories from '../../../tests/factories'
 import { mockToolRequestsCopilot } from '../../../tests/helpers'
 import { WebsocketClient } from '../../../websockets/workers'
 import * as utils from '../../utils/progressTracker'
-import * as runDocumentAtCommitWithAutoToolResponses from './runDocumentAtCommitWithAutoToolResponses'
+import * as runDocumentAtCommitModule from '../../../services/commits'
 
 const incrementErrorsMock = vi.hoisted(() => vi.fn())
 
@@ -26,10 +26,7 @@ describe('runDocumentJob', () => {
       buildRedisConnection: vi.fn(),
     }
   })
-  vi.spyOn(
-    runDocumentAtCommitWithAutoToolResponses,
-    'runDocumentAtCommitWithAutoToolResponses',
-  )
+  vi.spyOn(runDocumentAtCommitModule, 'runDocumentAtCommit')
   // @ts-ignore
   vi.spyOn(utils, 'ProgressTracker').mockImplementation(() => ({
     incrementCompleted: vi.fn(),
@@ -80,25 +77,25 @@ describe('runDocumentJob', () => {
       messages: Promise.resolve([]),
       trace: factories.createTelemetryTrace({}),
     }
-    vi.mocked(
-      runDocumentAtCommitWithAutoToolResponses.runDocumentAtCommitWithAutoToolResponses,
-    ).mockResolvedValue(
+    vi.mocked(runDocumentAtCommitModule.runDocumentAtCommit).mockResolvedValue(
       // @ts-ignore
       Result.ok(mockResult),
     )
 
     await runDocumentJob(mockJob)
 
-    expect(
-      runDocumentAtCommitWithAutoToolResponses.runDocumentAtCommitWithAutoToolResponses,
-    ).toHaveBeenCalledWith({
+    expect(runDocumentAtCommitModule.runDocumentAtCommit).toHaveBeenCalledWith({
       context: expect.anything(),
-      workspaceId: workspace.id,
-      projectId: project.id,
-      documentUuid: document.documentUuid,
-      commitUuid: commit.uuid,
+      workspace: expect.objectContaining({ id: workspace.id }),
+      commit: expect.objectContaining({ uuid: commit.uuid }),
+      document: expect.objectContaining({
+        documentUuid: document.documentUuid,
+      }),
       parameters: { param1: 'value1' },
       source: LogSources.Playground,
+      simulationSettings: {
+        simulateToolResponses: true,
+      },
     })
 
     expect(WebsocketClient.sendEvent).toHaveBeenCalledWith(
@@ -121,9 +118,9 @@ describe('runDocumentJob', () => {
     const mod = await import('./runDocumentJob')
     const runDocumentJob = mod.runDocumentJob
 
-    vi.mocked(
-      runDocumentAtCommitWithAutoToolResponses.runDocumentAtCommitWithAutoToolResponses,
-    ).mockRejectedValue(new Error('Test error'))
+    vi.mocked(runDocumentAtCommitModule.runDocumentAtCommit).mockRejectedValue(
+      new Error('Test error'),
+    )
 
     await runDocumentJob(mockJob)
 
@@ -140,9 +137,7 @@ describe('runDocumentJob', () => {
       },
     )
 
-    expect(
-      runDocumentAtCommitWithAutoToolResponses.runDocumentAtCommitWithAutoToolResponses,
-    ).toHaveBeenCalled()
+    expect(runDocumentAtCommitModule.runDocumentAtCommit).toHaveBeenCalled()
     expect(incrementErrorsMock).toHaveBeenCalled()
   })
 
@@ -154,9 +149,9 @@ describe('runDocumentJob', () => {
       message: 'Rate limit error',
     })
 
-    vi.mocked(
-      runDocumentAtCommitWithAutoToolResponses.runDocumentAtCommitWithAutoToolResponses,
-    ).mockRejectedValue(rateLimitError)
+    vi.mocked(runDocumentAtCommitModule.runDocumentAtCommit).mockRejectedValue(
+      rateLimitError,
+    )
 
     await expect(runDocumentJob(mockJob)).rejects.toThrow('Rate limit error')