diff --git a/integrations/anthropic/src/actions/generate-content.ts b/integrations/anthropic/src/actions/generate-content.ts index 1fc2327964d..db6c9edd847 100644 --- a/integrations/anthropic/src/actions/generate-content.ts +++ b/integrations/anthropic/src/actions/generate-content.ts @@ -32,7 +32,7 @@ export async function generateContent( input.reasoningEffort = 'medium' } - // TODO: Uncomment this when we have removed the "reasoning" model IDs from the model list. + // TODO: Uncomment this when we have removed the fake "reasoning" model IDs from the model list. // logger // .forBot() // .warn( diff --git a/integrations/anthropic/src/index.ts b/integrations/anthropic/src/index.ts index f28e23da8ed..a574b617733 100644 --- a/integrations/anthropic/src/index.ts +++ b/integrations/anthropic/src/index.ts @@ -51,7 +51,7 @@ const LanguageModels: Record = { name: 'Claude Sonnet 4 (Reasoning Mode)', description: 'This model uses the "Extended Thinking" mode and will use a significantly higher amount of output tokens than the Standard Mode, so this model should only be used for tasks that actually require it.\n\nClaude Sonnet 4 significantly enhances the capabilities of its predecessor, Sonnet 3.7, excelling in both coding and reasoning tasks with improved precision and controllability. Sonnet 4 balances capability and computational efficiency, making it suitable for a broad range of applications from routine coding tasks to complex software development projects. Key enhancements include improved autonomous codebase navigation, reduced error rates in agent-driven workflows, and increased reliability in following intricate instructions.', - tags: ['deprecated', 'vision', 'reasoning', 'general-purpose', 'agents', 'coding'], + tags: ['vision', 'reasoning', 'general-purpose', 'agents', 'coding'], input: { costPer1MTokens: 3, maxTokens: 200_000, @@ -79,7 +79,7 @@ const LanguageModels: Record = { name: 'Claude 3.7 Sonnet (Reasoning Mode)', description: 'This model uses the "Extended Thinking" mode and will use a significantly higher amount of output tokens than the Standard Mode, so this model should only be used for tasks that actually require it.\n\nClaude 3.7 Sonnet is an advanced large language model with improved reasoning, coding, and problem-solving capabilities. The model demonstrates notable improvements in coding, particularly in front-end development and full-stack updates, and excels in agentic workflows, where it can autonomously navigate multi-step processes.', - tags: ['deprecated', 'vision', 'reasoning', 'general-purpose', 'agents', 'coding'], + tags: ['vision', 'reasoning', 'general-purpose', 'agents', 'coding'], input: { costPer1MTokens: 3, maxTokens: 200_000, diff --git a/packages/cli/package.json b/packages/cli/package.json index 11834c47029..94e3a3df944 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@botpress/cli", - "version": "4.17.1", + "version": "4.17.2", "description": "Botpress CLI", "scripts": { "build": "pnpm run bundle && pnpm run template:gen", diff --git a/packages/cli/src/command-implementations/add-command.ts b/packages/cli/src/command-implementations/add-command.ts index 10e892a1236..b57072be7e6 100644 --- a/packages/cli/src/command-implementations/add-command.ts +++ b/packages/cli/src/command-implementations/add-command.ts @@ -1,6 +1,7 @@ import * as sdk from '@botpress/sdk' import * as fslib from 'fs' import * as pathlib from 'path' +import semver from 'semver' import * as apiUtils from '../api' import * as codegen from '../code-generation' import type commandDefinitions from '../command-definitions' @@ -113,7 +114,17 @@ export class AddCommand extends GlobalCommand { await this._uninstall(installPath) } - if (ref.type === 'name') { + if (ref.type === 'name' && ref.version === pkgRef.LATEST_TAG) { + // If the semver version expression is 'latest', we assume the project + // is compatible with all versions of the latest major: + targetPackage.pkg.version = `^${semver.major(targetPackage.pkg.version)}.0.0` + + this.logger.log( + `Dependency "${packageName}" will be installed with version "${targetPackage.pkg.version}". ` + + `To pin a specific version or version range, please change "${targetPackage.type}:${packageName}@latest" ` + + 'to a specific version number or range instead of "latest".' + ) + } else if (ref.type === 'name') { // Preserve the semver version expression in the generated code: targetPackage.pkg.version = ref.version } diff --git a/packages/cli/src/package-ref.ts b/packages/cli/src/package-ref.ts index f65854d5206..129381bdda3 100644 --- a/packages/cli/src/package-ref.ts +++ b/packages/cli/src/package-ref.ts @@ -30,7 +30,7 @@ export type LocalPackageRef = { export type ApiPackageRef = UUIDPackageRef | NamePackageRef export type PackageRef = ApiPackageRef | LocalPackageRef -const LATEST_TAG = 'latest' +export const LATEST_TAG = 'latest' export const formatPackageRef = (ref: PackageRef): string => { if (ref.type === 'path') { diff --git a/packages/sdk/src/integration/client/index.ts b/packages/sdk/src/integration/client/index.ts index 185edad8c05..912cb60f4b0 100644 --- a/packages/sdk/src/integration/client/index.ts +++ b/packages/sdk/src/integration/client/index.ts @@ -54,9 +54,9 @@ export class IntegrationSpecificClient = ((x) => this._client.getOrCreateMessage(x)) as types.GetOrCreateMessage public getMessage: types.GetMessage = ((x) => - this._client.getMessage(x)) as types.GetMessage + this._client.getMessage(x).then((r) => r)) as types.GetMessage public updateMessage: types.UpdateMessage = ((x) => - this._client.updateMessage(x)) as types.UpdateMessage + this._client.updateMessage(x).then((r) => r)) as types.UpdateMessage public listMessages: types.ListMessages = ((x) => this._client.listMessages(x)) as types.ListMessages public deleteMessage: types.DeleteMessage = ((x) => diff --git a/packages/sdk/src/integration/client/sub-types.test.ts b/packages/sdk/src/integration/client/sub-types.test.ts index c876abf384d..0a360153218 100644 --- a/packages/sdk/src/integration/client/sub-types.test.ts +++ b/packages/sdk/src/integration/client/sub-types.test.ts @@ -61,6 +61,53 @@ test('TagsOfMessage with specific message', () => { type _assertion = utils.AssertTrue> }) +test('EnumerateMessages of FooBarBazIntegration returns union of all message types', () => { + type Actual = EnumerateMessages + type Expected = { + messageFoo: { + type: 'messageFoo' + tags: { + fooMessageTag1?: string + fooMessageTag2?: string + fooMessageTag3?: string + } + payload: { + foo: string + } + } + messageBar: { + type: 'messageBar' + tags: { + barMessageTag1?: string + barMessageTag2?: string + barMessageTag3?: string + } + payload: { + bar: number + } + } + messageBaz: { + type: 'messageBaz' + tags: { + bazMessageTag1?: string + bazMessageTag2?: string + bazMessageTag3?: string + } + payload: { + baz: boolean + } + } + } + + type _assertion = utils.AssertAll< + [ + utils.AssertExtends, + utils.AssertExtends, + utils.AssertTrue>, + ] + > +}) + test('GetChannelByName', () => { type Actual = GetChannelByName type Expected = { @@ -90,10 +137,11 @@ test('GetChannelByName', () => { test('GetMessageByName', () => { type Actual = GetMessageByName type Expected = { + type: 'messageFoo' tags: { - fooMessageTag1: '' - fooMessageTag2: '' - fooMessageTag3: '' + fooMessageTag1?: string + fooMessageTag2?: string + fooMessageTag3?: string } payload: { foo: string diff --git a/packages/sdk/src/integration/client/sub-types.ts b/packages/sdk/src/integration/client/sub-types.ts index f25e6601a7a..0ce38a40e68 100644 --- a/packages/sdk/src/integration/client/sub-types.ts +++ b/packages/sdk/src/integration/client/sub-types.ts @@ -5,7 +5,10 @@ export type EnumerateMessages = uti utils.ValueOf<{ [TChannelName in keyof TIntegration['channels']]: { [TMessageName in keyof TIntegration['channels'][TChannelName]['messages']]: { - tags: TIntegration['channels'][TChannelName]['message']['tags'] + type: TMessageName + tags: { + [Tag in keyof TIntegration['channels'][TChannelName]['message']['tags']]?: string + } payload: TIntegration['channels'][TChannelName]['messages'][TMessageName] } } @@ -23,6 +26,7 @@ export type GetMessageByName< > = utils.Cast< EnumerateMessages[TMessageName], { + type: string tags: Record payload: any } diff --git a/packages/sdk/src/integration/client/types.test.ts b/packages/sdk/src/integration/client/types.test.ts index 9b706a51469..c7b7fc7291a 100644 --- a/packages/sdk/src/integration/client/types.test.ts +++ b/packages/sdk/src/integration/client/types.test.ts @@ -5,6 +5,21 @@ import * as utils from '../../utils/type-utils' import * as types from './types' import { FooBarBazIntegration } from '../../fixtures' +/** + * Utility type to modify the response of an operation to have optional tags + */ +type _WithOptionalTagResponse Promise<{ message: { tags: Record } }>> = + (...x: Parameters) => Promise< + utils.Normalize< + utils.Merge< + Awaited>, + { + message: utils.Merge>['message'], { tags: Record }> + } + > + > + > + const _mockClient = () => new Proxy>({} as any, { get: () => { @@ -82,22 +97,22 @@ describe.concurrent('ClientOperations', () => { }) test('createMessage of IntegrationSpecificClient extends general', () => { type Specific = types.ClientOperations['createMessage'] - type General = client.Client['createMessage'] + type General = _WithOptionalTagResponse type _assertion = utils.AssertExtends }) test('getOrCreateMessage of IntegrationSpecificClient extends general', () => { type Specific = types.ClientOperations['getOrCreateMessage'] - type General = client.Client['getOrCreateMessage'] + type General = _WithOptionalTagResponse type _assertion = utils.AssertExtends }) test('getMessage of IntegrationSpecificClient extends general', () => { type Specific = types.ClientOperations['getMessage'] - type General = client.Client['getMessage'] + type General = _WithOptionalTagResponse type _assertion = utils.AssertExtends }) test('updateMessage of IntegrationSpecificClient extends general', () => { type Specific = types.ClientOperations['updateMessage'] - type General = client.Client['updateMessage'] + type General = _WithOptionalTagResponse type _assertion = utils.AssertExtends }) test('listMessages of IntegrationSpecificClient extends general', () => { @@ -198,17 +213,63 @@ describe.concurrent('ClientOperations', () => { test('getMessage response should include all possible message tags', () => { type Actual = types.ClientOutputs['getMessage']['message']['tags'] + type Expected = + | { + fooMessageTag1?: string | undefined + fooMessageTag2?: string | undefined + fooMessageTag3?: string | undefined + } + | { + barMessageTag1?: string | undefined + barMessageTag2?: string | undefined + barMessageTag3?: string | undefined + } + | { + bazMessageTag1?: string | undefined + bazMessageTag2?: string | undefined + bazMessageTag3?: string | undefined + } + type _assertion = utils.AssertTrue> + }) + + test('getMessage response should be a union of all possible payloads', () => { + type Actual = types.ClientOutputs['getMessage']['message'] type Expected = { - fooMessageTag1?: string | undefined - fooMessageTag2?: string | undefined - fooMessageTag3?: string | undefined - barMessageTag1?: string | undefined - barMessageTag2?: string | undefined - barMessageTag3?: string | undefined - bazMessageTag1?: string | undefined - bazMessageTag2?: string | undefined - bazMessageTag3?: string | undefined - } + id: string + createdAt: string + updatedAt: string + direction: 'incoming' | 'outgoing' + userId: string + conversationId: string + } & ( + | { + type: 'messageFoo' + payload: { foo: string } + tags: { + fooMessageTag1?: string | undefined + fooMessageTag2?: string | undefined + fooMessageTag3?: string | undefined + } + } + | { + type: 'messageBar' + payload: { bar: number } + tags: { + barMessageTag1?: string | undefined + barMessageTag2?: string | undefined + barMessageTag3?: string | undefined + } + } + | { + type: 'messageBaz' + payload: { baz: boolean } + tags: { + bazMessageTag1?: string | undefined + bazMessageTag2?: string | undefined + bazMessageTag3?: string | undefined + } + } + ) type _assertion = utils.AssertTrue> }) diff --git a/packages/sdk/src/integration/client/types.ts b/packages/sdk/src/integration/client/types.ts index 9a4e77f7e7f..e943d4a840b 100644 --- a/packages/sdk/src/integration/client/types.ts +++ b/packages/sdk/src/integration/client/types.ts @@ -1,14 +1,7 @@ import * as client from '@botpress/client' import * as utils from '../../utils/type-utils' import * as common from '../common' -import { - EnumerateMessages, - ConversationTags, - GetChannelByName, - GetMessageByName, - MessageTags, - TagsOfMessage, -} from './sub-types' +import { EnumerateMessages, ConversationTags, GetChannelByName, GetMessageByName, MessageTags } from './sub-types' type Arg any> = Parameters[number] type Res any> = ReturnType @@ -119,12 +112,8 @@ type MessageResponse< TMessage extends keyof EnumerateMessages = keyof EnumerateMessages, > = { message: utils.Merge< - Awaited>['message'], - { - type: utils.Cast - payload: GetMessageByName['payload'] - tags: common.ToTags> - } + Awaited>['message'], + utils.Cast[TMessage], object> > } diff --git a/plugins/file-synchronizer/plugin.definition.ts b/plugins/file-synchronizer/plugin.definition.ts index 56b27bea339..c8cb894e4a9 100644 --- a/plugins/file-synchronizer/plugin.definition.ts +++ b/plugins/file-synchronizer/plugin.definition.ts @@ -59,7 +59,7 @@ const FILE_FILTER_PROPS = sdk.z.object({ export default new sdk.PluginDefinition({ name: 'file-synchronizer', - version: '0.7.6', + version: '0.7.7', title: 'File Synchronizer', description: 'Synchronize files from external services to Botpress', icon: 'icon.svg', diff --git a/plugins/file-synchronizer/src/hooks/workflow-continued/build-queue.ts b/plugins/file-synchronizer/src/hooks/workflow-continued/build-queue.ts index b8fcf04ebea..9a78145a8dd 100644 --- a/plugins/file-synchronizer/src/hooks/workflow-continued/build-queue.ts +++ b/plugins/file-synchronizer/src/hooks/workflow-continued/build-queue.ts @@ -20,6 +20,18 @@ export const handleEvent: bp.WorkflowHandlers['buildQueue'] = async (props) => { return } + const { syncQueue: finalSyncQueue } = await SyncQueue.jobFileManager.getSyncQueue(props, workflowState.jobFileId) + + if (finalSyncQueue.length === 0) { + props.logger.info( + 'Enumeration matched no files. Nothing to sync. Please check your include and exclude rules. ' + + `There could also be an issue with your configuration in the "${props.interfaces['files-readonly'].name}" integration. ` + + 'For example, your access token might be missing some permissions or the integration might not be set up correctly.' + ) + await props.workflow.setCompleted() + return + } + props.logger.info('Enumeration completed. Starting sync job...') await props.workflow.setCompleted() await props.workflows.processQueue.startNewInstance({ @@ -40,6 +52,8 @@ const _getEnumerateAllFilesRecursiveProps = ( ): SyncQueue.directoryTraversalWithBatching.EnumerateAllFilesRecursiveProps => ({ ...props, + configuration: props.workflow.input, + currentEnumerationState: workflowState.enumerationState, integration: { listItemsInFolder: props.actions['files-readonly'].listItemsInFolder }, diff --git a/plugins/file-synchronizer/src/sync-queue/directory-traversal-with-batching.test.ts b/plugins/file-synchronizer/src/sync-queue/directory-traversal-with-batching.test.ts index d0342c13e34..4fb8f231629 100644 --- a/plugins/file-synchronizer/src/sync-queue/directory-traversal-with-batching.test.ts +++ b/plugins/file-synchronizer/src/sync-queue/directory-traversal-with-batching.test.ts @@ -242,7 +242,7 @@ describe.concurrent('enumerateAllFilesRecursive', () => { id: 'file1', }), ]) - expect(mocks.logger.debug).toHaveBeenCalledWith('Ignoring item', expect.any(Object)) + expect(mocks.logger.debug).toHaveBeenCalledWith('Ignoring item', expect.any(Object), expect.any(String)) }) }) }) diff --git a/plugins/file-synchronizer/src/sync-queue/directory-traversal-with-batching.ts b/plugins/file-synchronizer/src/sync-queue/directory-traversal-with-batching.ts index 9dfdf89e6cd..82b8ca72a1d 100644 --- a/plugins/file-synchronizer/src/sync-queue/directory-traversal-with-batching.ts +++ b/plugins/file-synchronizer/src/sync-queue/directory-traversal-with-batching.ts @@ -52,7 +52,11 @@ export const enumerateAllFilesRecursive = async ({ const globMatchResult = globMatcher.matchItem({ configuration, item: folderItem, itemPath }) if (globMatchResult.shouldBeIgnored) { - logger.debug('Ignoring item', { itemPath, reason: globMatchResult.reason }) + logger.debug( + 'Ignoring item', + { itemPath, reason: globMatchResult.reason }, + JSON.stringify({ item: folderItem, configuration }) + ) continue } diff --git a/plugins/file-synchronizer/src/sync-queue/queue-batching.ts b/plugins/file-synchronizer/src/sync-queue/queue-batching.ts index ec459551514..fca4632eced 100644 --- a/plugins/file-synchronizer/src/sync-queue/queue-batching.ts +++ b/plugins/file-synchronizer/src/sync-queue/queue-batching.ts @@ -4,12 +4,13 @@ import type * as types from '../types' type SimpleFile = Pick export const findBatchEndCursor = ({ - startCursor, + startCursor: unsafeStartCursor, files, }: { startCursor: number files: SimpleFile[] }): { endCursor: number } => { + const startCursor = Math.min(Math.max(unsafeStartCursor, 0), files.length - 1) let currentBatchSize = 0 let endCursor = files.length diff --git a/plugins/file-synchronizer/src/sync-queue/queue-processor.ts b/plugins/file-synchronizer/src/sync-queue/queue-processor.ts index 0932979b605..d760b36e89d 100644 --- a/plugins/file-synchronizer/src/sync-queue/queue-processor.ts +++ b/plugins/file-synchronizer/src/sync-queue/queue-processor.ts @@ -24,8 +24,20 @@ export type ProcessQueueProps = { } export const processQueue = async (props: ProcessQueueProps) => { + // Short-circuit if the sync queue is empty: + if (!props.syncQueue?.length) { + props.logger.info('Sync queue is empty. Nothing to process.') + return { finished: 'all' } as const + } + const syncQueue = structuredClone(props.syncQueue) as types.SyncQueue const startCursor = syncQueue.findIndex((file) => file.status === 'pending') ?? syncQueue.length - 1 + + if (startCursor < 0) { + props.logger.info('No files left to process in the sync queue.') + return { finished: 'all' } as const + } + const { endCursor } = findBatchEndCursor({ startCursor, files: syncQueue }) const filesInBatch = syncQueue.slice(startCursor, endCursor)