Simulation Environment

apps/gateway/src/services/cloudwatchMetrics.ts

@@ -24,7 +24,11 @@ class CloudWatchMetricsService {
   }
 
   private isConfigured(): boolean {
-    return Boolean(env.AWS_ACCESS_KEY && env.AWS_ACCESS_SECRET)
+    return Boolean(
+      env.NODE_ENV === 'production' &&
+        env.AWS_ACCESS_KEY &&
+        env.AWS_ACCESS_SECRET,
+    )
   }
 
   incrementInflightRequests(): void {

apps/web/src/components/ChatWrapper/Message/Content/ToolCall/LatitudeTools/Code.tsx

@@ -11,11 +11,56 @@ import {
   ToolCardWrapper,
 } from '../_components/ToolCard'
 import { ToolCardHeader } from '../_components/ToolCard/Header'
-import { ToolCardContentWrapper } from '../_components/ToolCard/Content'
-import { Alert } from '@latitude-data/web-ui/atoms/Alert'
+import {
+  ToolCardContentWrapper,
+  ToolCardOutput,
+} from '../_components/ToolCard/Content'
 import { Icon } from '@latitude-data/web-ui/atoms/Icons'
 import { Text } from '@latitude-data/web-ui/atoms/Text'
 
+const isExpectedOutput = (toolResponse: ToolContent | undefined) => {
+  // Returns false if the tool response does not contain the expected output
+  if (!toolResponse) return false
+  if (toolResponse.isError) return false
+
+  if (typeof toolResponse.result !== 'string') return false
+  return true
+}
+
+function RunCodeOutput({
+  toolResponse,
+  simulated,
+}: {
+  toolResponse: ToolContent | undefined
+  simulated?: boolean
+}) {
+  const isExpectedResponse = useMemo(
+    () => isExpectedOutput(toolResponse),
+    [toolResponse],
+  )
+
+  if (!toolResponse) {
+    return (
+      <ToolCardContentWrapper>
+        <div className='flex flex-row gap-2 items-center justify-center pb-3'>
+          <Icon name='loader' color='foregroundMuted' spin />
+          <Text.H5 color='foregroundMuted'>Running code...</Text.H5>
+        </div>
+      </ToolCardContentWrapper>
+    )
+  }
+
+  if (!isExpectedResponse) {
+    return <ToolCardOutput toolResponse={toolResponse} simulated={simulated} />
+  }
+
+  return (
+    <ToolCardContentWrapper>
+      <CodeBlock language='shell'>{toolResponse.result as string}</CodeBlock>
+    </ToolCardContentWrapper>
+  )
+}
+
 function runCodeContent(args: CodeToolArgs): string {
   if (!args.dependencies) return args.code
 
@@ -49,34 +94,17 @@ export function RunCodeLatitudeToolCard({
         status={status}
         isOpen={isOpen}
         onToggle={() => setIsOpen(!isOpen)}
+        simulated={toolRequest._sourceData?.simulated}
       />
       {isOpen && (
         <>
           <ToolCardContentWrapper>
             <CodeBlock language={args.language}>{value}</CodeBlock>
           </ToolCardContentWrapper>
-          <ToolCardContentWrapper badge='Output'>
-            {toolResponse ? (
-              toolResponse.isError ? (
-                <div className='w-full pt-3 items-center'>
-                  <Alert
-                    variant='destructive'
-                    title='Error'
-                    description={JSON.stringify(toolResponse.result, null, 2)}
-                  />
-                </div>
-              ) : (
-                <CodeBlock language='shell'>
-                  {toolResponse.result as string}
-                </CodeBlock>
-              )
-            ) : (
-              <div className='flex flex-row gap-2 items-center justify-center pb-3'>
-                <Icon name='loader' color='foregroundMuted' spin />
-                <Text.H5 color='foregroundMuted'>Running code...</Text.H5>
-              </div>
-            )}
-          </ToolCardContentWrapper>
+          <RunCodeOutput
+            toolResponse={toolResponse}
+            simulated={toolRequest._sourceData?.simulated}
+          />
         </>
       )}
     </ToolCardWrapper>

apps/web/src/components/ChatWrapper/Message/Content/ToolCall/LatitudeTools/Extract.tsx

@@ -16,10 +16,73 @@ import {
   ToolCardWrapper,
 } from '../_components/ToolCard'
 import { ToolCardHeader } from '../_components/ToolCard/Header'
-import { ToolCardContentWrapper } from '../_components/ToolCard/Content'
-import { Alert } from '@latitude-data/web-ui/atoms/Alert'
+import {
+  ToolCardContentWrapper,
+  ToolCardOutput,
+} from '../_components/ToolCard/Content'
 import { Icon } from '@latitude-data/web-ui/atoms/Icons'
 
+const isExpectedOutput = (toolResponse: ToolContent | undefined) => {
+  // Returns false if the tool response does not contain the expected output
+  if (!toolResponse) return false
+  if (toolResponse.isError) return false
+
+  if (typeof toolResponse.result !== 'object' || toolResponse.result === null) {
+    return false
+  }
+
+  if (!('content' in toolResponse.result)) return false
+  const { content } = toolResponse.result
+  if (typeof content !== 'string') return false
+
+  return true
+}
+
+function WebExtractOutput({
+  toolResponse,
+  simulated,
+}: {
+  toolResponse: ToolContent | undefined
+  simulated?: boolean
+}) {
+  const isExpectedResponse = useMemo(
+    () => isExpectedOutput(toolResponse),
+    [toolResponse],
+  )
+
+  const markdownContent = useMemo(() => {
+    if (!isExpectedResponse) return undefined
+    return (toolResponse!.result as ExtractToolResult).content
+  }, [toolResponse, isExpectedResponse])
+
+  if (!toolResponse) {
+    return (
+      <ToolCardContentWrapper>
+        <div className='flex flex-row gap-2 items-center justify-center pb-3'>
+          <Icon name='loader' color='foregroundMuted' spin />
+          <Text.H5 color='foregroundMuted'>Loading page...</Text.H5>
+        </div>
+      </ToolCardContentWrapper>
+    )
+  }
+
+  if (!isExpectedResponse) {
+    return <ToolCardOutput toolResponse={toolResponse} simulated={simulated} />
+  }
+
+  return (
+    <ToolCardContentWrapper>
+      {markdownContent ? (
+        <Markdown size='sm' color='foregroundMuted'>
+          {markdownContent}
+        </Markdown>
+      ) : (
+        <Text.H5 color='foregroundMuted'>No content</Text.H5>
+      )}
+    </ToolCardContentWrapper>
+  )
+}
+
 export function WebExtractLatitudeToolCard({
   toolRequest,
   toolResponse,
@@ -30,12 +93,6 @@ export function WebExtractLatitudeToolCard({
   status: 'pending' | 'success' | 'error'
 }) {
   const [isOpen, setIsOpen] = useState(false)
-
-  const markdownContent = useMemo(() => {
-    if (!toolResponse || toolResponse.isError) return undefined
-    return (toolResponse.result as ExtractToolResult).content
-  }, [toolResponse])
-
   const args = toolRequest.args as ExtractToolArgs
 
   return (
@@ -46,36 +103,13 @@ export function WebExtractLatitudeToolCard({
         status={status}
         isOpen={isOpen}
         onToggle={() => setIsOpen(!isOpen)}
+        simulated={toolRequest._sourceData?.simulated}
       />
       {isOpen && (
-        <ToolCardContentWrapper>
-          {toolResponse ? (
-            toolResponse.isError ? (
-              <div className='w-full pt-3 items-center'>
-                <Alert
-                  variant='destructive'
-                  title='Error'
-                  description={JSON.stringify(toolResponse.result, null, 2)}
-                />
-              </div>
-            ) : (
-              <div className='flex flex-col gap-4'>
-                {markdownContent ? (
-                  <Markdown size='sm' color='primary'>
-                    {markdownContent}
-                  </Markdown>
-                ) : (
-                  <Text.H5 color='foregroundMuted'>No content</Text.H5>
-                )}
-              </div>
-            )
-          ) : (
-            <div className='flex flex-row gap-2 items-center justify-center pb-3'>
-              <Icon name='loader' color='foregroundMuted' spin />
-              <Text.H5 color='foregroundMuted'>Loading page...</Text.H5>
-            </div>
-          )}
-        </ToolCardContentWrapper>
+        <WebExtractOutput
+          toolResponse={toolResponse}
+          simulated={toolRequest._sourceData?.simulated}
+        />
       )}
     </ToolCardWrapper>
   )

apps/web/src/components/ChatWrapper/Message/Content/ToolCall/LatitudeTools/Search.tsx

@@ -16,8 +16,10 @@ import {
   ToolCardWrapper,
 } from '../_components/ToolCard'
 import { ToolCardHeader } from '../_components/ToolCard/Header'
-import { ToolCardContentWrapper } from '../_components/ToolCard/Content'
-import { Alert } from '@latitude-data/web-ui/atoms/Alert'
+import {
+  ToolCardContentWrapper,
+  ToolCardOutput,
+} from '../_components/ToolCard/Content'
 
 function topicText({
   topic,
@@ -93,6 +95,87 @@ function WebSearchResult({
   )
 }
 
+const isExpectedOutput = (toolResponse: ToolContent | undefined) => {
+  // Returns false if the tool response does not contain the expected output
+  if (!toolResponse) return false
+  if (toolResponse.isError) return false
+
+  if (typeof toolResponse.result !== 'object' || toolResponse.result === null) {
+    return false
+  }
+
+  if (!('results' in toolResponse.result)) return false
+  const results = toolResponse.result.results
+
+  if (!Array.isArray(results)) return false
+  if (results.length === 0) return true
+
+  if (
+    results.some(
+      (result) =>
+        typeof result !== 'object' ||
+        result === null ||
+        !('title' in result) ||
+        typeof result.title !== 'string' ||
+        !('url' in result) ||
+        typeof result.url !== 'string' ||
+        !('content' in result) ||
+        typeof result.content !== 'string',
+    )
+  ) {
+    return false
+  }
+
+  return true
+}
+
+function WebSearchOutput({
+  toolResponse,
+  simulated,
+}: {
+  toolResponse: ToolContent | undefined
+  simulated?: boolean
+}) {
+  const isExpectedResponse = useMemo(
+    () => isExpectedOutput(toolResponse),
+    [toolResponse],
+  )
+
+  const searchResults = useMemo(() => {
+    if (!isExpectedResponse) return []
+    return (toolResponse!.result as SearchToolResult).results
+  }, [toolResponse, isExpectedResponse])
+
+  if (!toolResponse) {
+    return (
+      <ToolCardContentWrapper>
+        <div className='flex flex-row gap-2 items-center justify-center pb-3'>
+          <Icon name='loader' color='foregroundMuted' spin />
+          <Text.H5 color='foregroundMuted'>Searching...</Text.H5>
+        </div>
+      </ToolCardContentWrapper>
+    )
+  }
+
+  if (!isExpectedResponse) {
+    return <ToolCardOutput toolResponse={toolResponse} simulated={simulated} />
+  }
+
+  return (
+    <ToolCardContentWrapper>
+      <div className='flex flex-col gap-4'>
+        {searchResults.length ? (
+          searchResults.map((result, index) => (
+            <WebSearchResult key={index} result={result} />
+          ))
+        ) : (
+          <Text.H5 color='foregroundMuted'>No results found</Text.H5>
+        )}
+      </div>
+    </ToolCardContentWrapper>
+  )
+}
+
 export function WebSearchLatitudeToolCard({
   toolRequest,
   toolResponse,
@@ -103,12 +186,6 @@ export function WebSearchLatitudeToolCard({
   status: 'pending' | 'success' | 'error'
 }) {
   const [isOpen, setIsOpen] = useState(false)
-
-  const searchResults = useMemo(() => {
-    if (!toolResponse || toolResponse.isError) return []
-    return (toolResponse.result as SearchToolResult).results
-  }, [toolResponse])
-
   const args = toolRequest.args as SearchToolArgs
 
   return (
@@ -124,36 +201,13 @@ export function WebSearchLatitudeToolCard({
         status={status}
         isOpen={isOpen}
         onToggle={() => setIsOpen(!isOpen)}
+        simulated={toolRequest._sourceData?.simulated}
       />
       {isOpen && (
-        <ToolCardContentWrapper>
-          {toolResponse ? (
-            toolResponse.isError ? (
-              <div className='w-full pt-3 items-center'>
-                <Alert
-                  variant='destructive'
-                  title='Error'
-                  description={JSON.stringify(toolResponse.result, null, 2)}
-                />
-              </div>
-            ) : (
-              <div className='flex flex-col gap-4'>
-                {searchResults.length ? (
-                  searchResults.map((result, index) => (
-                    <WebSearchResult key={index} result={result} />
-                  ))
-                ) : (
-                  <Text.H5 color='foregroundMuted'>No results found</Text.H5>
-                )}
-              </div>
-            )
-          ) : (
-            <div className='flex flex-row gap-2 items-center justify-center pb-3'>
-              <Icon name='loader' color='foregroundMuted' spin />
-              <Text.H5 color='foregroundMuted'>Searching...</Text.H5>
-            </div>
-          )}
-        </ToolCardContentWrapper>
+        <WebSearchOutput
+          toolResponse={toolResponse}
+          simulated={toolRequest._sourceData?.simulated}
+        />
       )}
     </ToolCardWrapper>
   )