[pull] master from botpress:master

integrations/anthropic/src/actions/generate-content.ts

@@ -32,7 +32,7 @@ export async function generateContent(
       input.reasoningEffort = 'medium'
     }
 
-    // TODO: Uncomment this when we have removed the "reasoning" model IDs from the model list.
+    // TODO: Uncomment this when we have removed the fake "reasoning" model IDs from the model list.
     // logger
     //   .forBot()
     //   .warn(

integrations/anthropic/src/index.ts

@@ -51,7 +51,7 @@ const LanguageModels: Record<ModelId, llm.ModelDetails> = {
     name: 'Claude Sonnet 4 (Reasoning Mode)',
     description:
       'This model uses the "Extended Thinking" mode and will use a significantly higher amount of output tokens than the Standard Mode, so this model should only be used for tasks that actually require it.\n\nClaude Sonnet 4 significantly enhances the capabilities of its predecessor, Sonnet 3.7, excelling in both coding and reasoning tasks with improved precision and controllability. Sonnet 4 balances capability and computational efficiency, making it suitable for a broad range of applications from routine coding tasks to complex software development projects. Key enhancements include improved autonomous codebase navigation, reduced error rates in agent-driven workflows, and increased reliability in following intricate instructions.',
-    tags: ['deprecated', 'vision', 'reasoning', 'general-purpose', 'agents', 'coding'],
+    tags: ['vision', 'reasoning', 'general-purpose', 'agents', 'coding'],
     input: {
       costPer1MTokens: 3,
       maxTokens: 200_000,
@@ -79,7 +79,7 @@ const LanguageModels: Record<ModelId, llm.ModelDetails> = {
     name: 'Claude 3.7 Sonnet (Reasoning Mode)',
     description:
       'This model uses the "Extended Thinking" mode and will use a significantly higher amount of output tokens than the Standard Mode, so this model should only be used for tasks that actually require it.\n\nClaude 3.7 Sonnet is an advanced large language model with improved reasoning, coding, and problem-solving capabilities. The model demonstrates notable improvements in coding, particularly in front-end development and full-stack updates, and excels in agentic workflows, where it can autonomously navigate multi-step processes.',
-    tags: ['deprecated', 'vision', 'reasoning', 'general-purpose', 'agents', 'coding'],
+    tags: ['vision', 'reasoning', 'general-purpose', 'agents', 'coding'],
     input: {
       costPer1MTokens: 3,
       maxTokens: 200_000,

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@botpress/cli",
-  "version": "4.17.1",
+  "version": "4.17.2",
   "description": "Botpress CLI",
   "scripts": {
     "build": "pnpm run bundle && pnpm run template:gen",

packages/cli/src/command-implementations/add-command.ts

@@ -1,6 +1,7 @@
 import * as sdk from '@botpress/sdk'
 import * as fslib from 'fs'
 import * as pathlib from 'path'
+import semver from 'semver'
 import * as apiUtils from '../api'
 import * as codegen from '../code-generation'
 import type commandDefinitions from '../command-definitions'
@@ -113,7 +114,17 @@ export class AddCommand extends GlobalCommand<AddCommandDefinition> {
       await this._uninstall(installPath)
     }
 
-    if (ref.type === 'name') {
+    if (ref.type === 'name' && ref.version === pkgRef.LATEST_TAG) {
+      // If the semver version expression is 'latest', we assume the project
+      // is compatible with all versions of the latest major:
+      targetPackage.pkg.version = `^${semver.major(targetPackage.pkg.version)}.0.0`
+
+      this.logger.log(
+        `Dependency "${packageName}" will be installed with version "${targetPackage.pkg.version}". ` +
+          `To pin a specific version or version range, please change "${targetPackage.type}:${packageName}@latest" ` +
+          'to a specific version number or range instead of "latest".'
+      )
+    } else if (ref.type === 'name') {
       // Preserve the semver version expression in the generated code:
       targetPackage.pkg.version = ref.version
     }

packages/cli/src/package-ref.ts

@@ -30,7 +30,7 @@ export type LocalPackageRef = {
 export type ApiPackageRef = UUIDPackageRef | NamePackageRef
 export type PackageRef = ApiPackageRef | LocalPackageRef
 
-const LATEST_TAG = 'latest'
+export const LATEST_TAG = 'latest'
 
 export const formatPackageRef = (ref: PackageRef): string => {
   if (ref.type === 'path') {

packages/sdk/src/integration/client/index.ts

@@ -54,9 +54,9 @@ export class IntegrationSpecificClient<TIntegration extends common.BaseIntegrati
   public getOrCreateMessage: types.GetOrCreateMessage<TIntegration> = ((x) =>
     this._client.getOrCreateMessage(x)) as types.GetOrCreateMessage<TIntegration>
   public getMessage: types.GetMessage<TIntegration> = ((x) =>
-    this._client.getMessage(x)) as types.GetMessage<TIntegration>
+    this._client.getMessage(x).then((r) => r)) as types.GetMessage<TIntegration>
   public updateMessage: types.UpdateMessage<TIntegration> = ((x) =>
-    this._client.updateMessage(x)) as types.UpdateMessage<TIntegration>
+    this._client.updateMessage(x).then((r) => r)) as types.UpdateMessage<TIntegration>
   public listMessages: types.ListMessages<TIntegration> = ((x) =>
     this._client.listMessages(x)) as types.ListMessages<TIntegration>
   public deleteMessage: types.DeleteMessage<TIntegration> = ((x) =>

packages/sdk/src/integration/client/sub-types.test.ts

@@ -61,6 +61,53 @@ test('TagsOfMessage with specific message', () => {
   type _assertion = utils.AssertTrue<utils.IsEqual<Actual, Expected>>
 })
 
+test('EnumerateMessages of FooBarBazIntegration returns union of all message types', () => {
+  type Actual = EnumerateMessages<FooBarBazIntegration>
+  type Expected = {
+    messageFoo: {
+      type: 'messageFoo'
+      tags: {
+        fooMessageTag1?: string
+        fooMessageTag2?: string
+        fooMessageTag3?: string
+      }
+      payload: {
+        foo: string
+      }
+    }
+    messageBar: {
+      type: 'messageBar'
+      tags: {
+        barMessageTag1?: string
+        barMessageTag2?: string
+        barMessageTag3?: string
+      }
+      payload: {
+        bar: number
+      }
+    }
+    messageBaz: {
+      type: 'messageBaz'
+      tags: {
+        bazMessageTag1?: string
+        bazMessageTag2?: string
+        bazMessageTag3?: string
+      }
+      payload: {
+        baz: boolean
+      }
+    }
+  }
+
+  type _assertion = utils.AssertAll<
+    [
+      utils.AssertExtends<Actual, Expected>,
+      utils.AssertExtends<Expected, Actual>,
+      utils.AssertTrue<utils.IsEqual<Actual, Expected>>,
+    ]
+  >
+})
+
 test('GetChannelByName', () => {
   type Actual = GetChannelByName<FooBarBazIntegration, 'channelFoo'>
   type Expected = {
@@ -90,10 +137,11 @@ test('GetChannelByName', () => {
 test('GetMessageByName', () => {
   type Actual = GetMessageByName<FooBarBazIntegration, 'messageFoo'>
   type Expected = {
+    type: 'messageFoo'
     tags: {
-      fooMessageTag1: ''
-      fooMessageTag2: ''
-      fooMessageTag3: ''
+      fooMessageTag1?: string
+      fooMessageTag2?: string
+      fooMessageTag3?: string
     }
     payload: {
       foo: string

packages/sdk/src/integration/client/sub-types.ts

@@ -5,7 +5,10 @@ export type EnumerateMessages<TIntegration extends common.BaseIntegration> = uti
   utils.ValueOf<{
     [TChannelName in keyof TIntegration['channels']]: {
       [TMessageName in keyof TIntegration['channels'][TChannelName]['messages']]: {
-        tags: TIntegration['channels'][TChannelName]['message']['tags']
+        type: TMessageName
+        tags: {
+          [Tag in keyof TIntegration['channels'][TChannelName]['message']['tags']]?: string
+        }
         payload: TIntegration['channels'][TChannelName]['messages'][TMessageName]
       }
     }
@@ -23,6 +26,7 @@ export type GetMessageByName<
 > = utils.Cast<
   EnumerateMessages<TIntegration>[TMessageName],
   {
+    type: string
     tags: Record<string, any>
     payload: any
   }

packages/sdk/src/integration/client/types.test.ts

@@ -5,6 +5,21 @@ import * as utils from '../../utils/type-utils'
 import * as types from './types'
 import { FooBarBazIntegration } from '../../fixtures'
 
+/**
+ * Utility type to modify the response of an operation to have optional tags
+ */
+type _WithOptionalTagResponse<TOperation extends (req: any) => Promise<{ message: { tags: Record<string, string> } }>> =
+  (...x: Parameters<TOperation>) => Promise<
+    utils.Normalize<
+      utils.Merge<
+        Awaited<ReturnType<TOperation>>,
+        {
+          message: utils.Merge<Awaited<ReturnType<TOperation>>['message'], { tags: Record<string, string | undefined> }>
+        }
+      >
+    >
+  >
+
 const _mockClient = <TIntegration extends BaseIntegration>() =>
   new Proxy<types.ClientOperations<TIntegration>>({} as any, {
     get: () => {
@@ -82,22 +97,22 @@ describe.concurrent('ClientOperations', () => {
   })
   test('createMessage of IntegrationSpecificClient extends general', () => {
     type Specific = types.ClientOperations<BaseIntegration>['createMessage']
-    type General = client.Client['createMessage']
+    type General = _WithOptionalTagResponse<client.Client['createMessage']>
     type _assertion = utils.AssertExtends<Specific, General>
   })
   test('getOrCreateMessage of IntegrationSpecificClient extends general', () => {
     type Specific = types.ClientOperations<BaseIntegration>['getOrCreateMessage']
-    type General = client.Client['getOrCreateMessage']
+    type General = _WithOptionalTagResponse<client.Client['getOrCreateMessage']>
     type _assertion = utils.AssertExtends<Specific, General>
   })
   test('getMessage of IntegrationSpecificClient extends general', () => {
     type Specific = types.ClientOperations<BaseIntegration>['getMessage']
-    type General = client.Client['getMessage']
+    type General = _WithOptionalTagResponse<client.Client['getMessage']>
     type _assertion = utils.AssertExtends<Specific, General>
   })
   test('updateMessage of IntegrationSpecificClient extends general', () => {
     type Specific = types.ClientOperations<BaseIntegration>['updateMessage']
-    type General = client.Client['updateMessage']
+    type General = _WithOptionalTagResponse<client.Client['updateMessage']>
     type _assertion = utils.AssertExtends<Specific, General>
   })
   test('listMessages of IntegrationSpecificClient extends general', () => {
@@ -198,17 +213,63 @@ describe.concurrent('ClientOperations', () => {
 
   test('getMessage response should include all possible message tags', () => {
     type Actual = types.ClientOutputs<FooBarBazIntegration>['getMessage']['message']['tags']
+    type Expected =
+      | {
+          fooMessageTag1?: string | undefined
+          fooMessageTag2?: string | undefined
+          fooMessageTag3?: string | undefined
+        }
+      | {
+          barMessageTag1?: string | undefined
+          barMessageTag2?: string | undefined
+          barMessageTag3?: string | undefined
+        }
+      | {
+          bazMessageTag1?: string | undefined
+          bazMessageTag2?: string | undefined
+          bazMessageTag3?: string | undefined
+        }
+    type _assertion = utils.AssertTrue<utils.IsEqual<Actual, Expected>>
+  })
+
+  test('getMessage response should be a union of all possible payloads', () => {
+    type Actual = types.ClientOutputs<FooBarBazIntegration>['getMessage']['message']
     type Expected = {
-      fooMessageTag1?: string | undefined
-      fooMessageTag2?: string | undefined
-      fooMessageTag3?: string | undefined
-      barMessageTag1?: string | undefined
-      barMessageTag2?: string | undefined
-      barMessageTag3?: string | undefined
-      bazMessageTag1?: string | undefined
-      bazMessageTag2?: string | undefined
-      bazMessageTag3?: string | undefined
-    }
+      id: string
+      createdAt: string
+      updatedAt: string
+      direction: 'incoming' | 'outgoing'
+      userId: string
+      conversationId: string
+    } & (
+      | {
+          type: 'messageFoo'
+          payload: { foo: string }
+          tags: {
+            fooMessageTag1?: string | undefined
+            fooMessageTag2?: string | undefined
+            fooMessageTag3?: string | undefined
+          }
+        }
+      | {
+          type: 'messageBar'
+          payload: { bar: number }
+          tags: {
+            barMessageTag1?: string | undefined
+            barMessageTag2?: string | undefined
+            barMessageTag3?: string | undefined
+          }
+        }
+      | {
+          type: 'messageBaz'
+          payload: { baz: boolean }
+          tags: {
+            bazMessageTag1?: string | undefined
+            bazMessageTag2?: string | undefined
+            bazMessageTag3?: string | undefined
+          }
+        }
+    )
     type _assertion = utils.AssertTrue<utils.IsEqual<Actual, Expected>>
   })
 

packages/sdk/src/integration/client/types.ts

@@ -1,14 +1,7 @@
 import * as client from '@botpress/client'
 import * as utils from '../../utils/type-utils'
 import * as common from '../common'
-import {
-  EnumerateMessages,
-  ConversationTags,
-  GetChannelByName,
-  GetMessageByName,
-  MessageTags,
-  TagsOfMessage,
-} from './sub-types'
+import { EnumerateMessages, ConversationTags, GetChannelByName, GetMessageByName, MessageTags } from './sub-types'
 
 type Arg<F extends (...args: any[]) => any> = Parameters<F>[number]
 type Res<F extends (...args: any[]) => any> = ReturnType<F>
@@ -119,12 +112,8 @@ type MessageResponse<
   TMessage extends keyof EnumerateMessages<TIntegration> = keyof EnumerateMessages<TIntegration>,
 > = {
   message: utils.Merge<
-    Awaited<Res<client.Client['createMessage']>>['message'],
-    {
-      type: utils.Cast<TMessage, string>
-      payload: GetMessageByName<TIntegration, TMessage>['payload']
-      tags: common.ToTags<TagsOfMessage<TIntegration, TMessage>>
-    }
+    Awaited<Res<client.Client['getMessage']>>['message'],
+    utils.Cast<EnumerateMessages<TIntegration>[TMessage], object>
   >
 }
 

plugins/file-synchronizer/plugin.definition.ts

@@ -59,7 +59,7 @@ const FILE_FILTER_PROPS = sdk.z.object({
 
 export default new sdk.PluginDefinition({
   name: 'file-synchronizer',
-  version: '0.7.6',
+  version: '0.7.7',
   title: 'File Synchronizer',
   description: 'Synchronize files from external services to Botpress',
   icon: 'icon.svg',

plugins/file-synchronizer/src/hooks/workflow-continued/build-queue.ts

@@ -20,6 +20,18 @@ export const handleEvent: bp.WorkflowHandlers['buildQueue'] = async (props) => {
     return
   }
 
+  const { syncQueue: finalSyncQueue } = await SyncQueue.jobFileManager.getSyncQueue(props, workflowState.jobFileId)
+
+  if (finalSyncQueue.length === 0) {
+    props.logger.info(
+      'Enumeration matched no files. Nothing to sync. Please check your include and exclude rules. ' +
+        `There could also be an issue with your configuration in the "${props.interfaces['files-readonly'].name}" integration. ` +
+        'For example, your access token might be missing some permissions or the integration might not be set up correctly.'
+    )
+    await props.workflow.setCompleted()
+    return
+  }
+
   props.logger.info('Enumeration completed. Starting sync job...')
   await props.workflow.setCompleted()
   await props.workflows.processQueue.startNewInstance({
@@ -40,6 +52,8 @@ const _getEnumerateAllFilesRecursiveProps = (
 ): SyncQueue.directoryTraversalWithBatching.EnumerateAllFilesRecursiveProps => ({
   ...props,
 
+  configuration: props.workflow.input,
+
   currentEnumerationState: workflowState.enumerationState,
   integration: { listItemsInFolder: props.actions['files-readonly'].listItemsInFolder },
 

plugins/file-synchronizer/src/sync-queue/directory-traversal-with-batching.test.ts

@@ -242,7 +242,7 @@ describe.concurrent('enumerateAllFilesRecursive', () => {
           id: 'file1',
         }),
       ])
-      expect(mocks.logger.debug).toHaveBeenCalledWith('Ignoring item', expect.any(Object))
+      expect(mocks.logger.debug).toHaveBeenCalledWith('Ignoring item', expect.any(Object), expect.any(String))
     })
   })
 })

plugins/file-synchronizer/src/sync-queue/directory-traversal-with-batching.ts

@@ -52,7 +52,11 @@ export const enumerateAllFilesRecursive = async ({
       const globMatchResult = globMatcher.matchItem({ configuration, item: folderItem, itemPath })
 
       if (globMatchResult.shouldBeIgnored) {
-        logger.debug('Ignoring item', { itemPath, reason: globMatchResult.reason })
+        logger.debug(
+          'Ignoring item',
+          { itemPath, reason: globMatchResult.reason },
+          JSON.stringify({ item: folderItem, configuration })
+        )
         continue
       }
 

plugins/file-synchronizer/src/sync-queue/queue-batching.ts

@@ -4,12 +4,13 @@ import type * as types from '../types'
 type SimpleFile = Pick<types.SyncQueueItem, 'sizeInBytes'>
 
 export const findBatchEndCursor = ({
-  startCursor,
+  startCursor: unsafeStartCursor,
   files,
 }: {
   startCursor: number
   files: SimpleFile[]
 }): { endCursor: number } => {
+  const startCursor = Math.min(Math.max(unsafeStartCursor, 0), files.length - 1)
   let currentBatchSize = 0
   let endCursor = files.length