diff --git a/packages/core/package.json b/packages/core/package.json index 88fe1c351..effb270ff 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -74,7 +74,6 @@ "dependencies": { "@anthropic-ai/sdk": "0.33.1", "@azure/identity": "4.5.0", - "@langchain/core": "0.3.26", "@midscene/recorder": "workspace:*", "@midscene/shared": "workspace:*", "@ui-tars/action-parser": "1.2.3", diff --git a/packages/core/src/ai-model/inspect.ts b/packages/core/src/ai-model/inspect.ts index 9173277f1..fc83c1bee 100644 --- a/packages/core/src/ai-model/inspect.ts +++ b/packages/core/src/ai-model/inspect.ts @@ -145,7 +145,7 @@ export async function AiLocateElement< targetElementDescription, 'cannot find the target element description', ); - const userInstructionPrompt = await findElementPrompt.format({ + const userInstructionPrompt = findElementPrompt({ pageDescription: description, targetElementDescription: extraTextFromUserPrompt(targetElementDescription), }); @@ -307,7 +307,7 @@ export async function AiLocateSection(options: { const { screenshotBase64 } = context; const systemPrompt = systemPromptToLocateSection(vlMode); - const sectionLocatorInstructionText = await sectionLocatorInstruction.format({ + const sectionLocatorInstructionText = sectionLocatorInstruction({ sectionDescription: extraTextFromUserPrompt(sectionDescription), }); const msgs: AIArgs = [ @@ -433,7 +433,7 @@ export async function AiExtractElementInfo< vlMode, }); - const extractDataPromptText = await extractDataQueryPrompt( + const extractDataPromptText = extractDataQueryPrompt( description, dataQuery, ); diff --git a/packages/core/src/ai-model/prompt/extraction.ts b/packages/core/src/ai-model/prompt/extraction.ts index cbf28e71d..1d5fb2b36 100644 --- a/packages/core/src/ai-model/prompt/extraction.ts +++ b/packages/core/src/ai-model/prompt/extraction.ts @@ -1,4 +1,3 @@ -import { PromptTemplate } from '@langchain/core/prompts'; import type { ResponseFormatJSONSchema } from 'openai/resources/index'; export function systemPromptToExtract() { @@ -87,7 +86,7 @@ By viewing the screenshot and page contents, you can extract the following data: `; } -export const extractDataQueryPrompt = async ( +export const extractDataQueryPrompt = ( pageDescription: string, dataQuery: string | Record, ) => { @@ -97,23 +96,16 @@ export const extractDataQueryPrompt = async ( } else { dataQueryText = JSON.stringify(dataQuery, null, 2); } - const extractDataPrompt = new PromptTemplate({ - template: ` + + return ` -{pageDescription} +${pageDescription} -{dataQuery} +${dataQueryText} - `, - inputVariables: ['pageDescription', 'dataQuery'], - }); - - return await extractDataPrompt.format({ - pageDescription, - dataQuery: dataQueryText, - }); + `; }; export const extractDataSchema: ResponseFormatJSONSchema = { diff --git a/packages/core/src/ai-model/prompt/llm-locator.ts b/packages/core/src/ai-model/prompt/llm-locator.ts index 0045e7dd4..507125da8 100644 --- a/packages/core/src/ai-model/prompt/llm-locator.ts +++ b/packages/core/src/ai-model/prompt/llm-locator.ts @@ -1,4 +1,3 @@ -import { PromptTemplate } from '@langchain/core/prompts'; import type { TVlModeTypes } from '@midscene/shared/env'; import type { ResponseFormatJSONSchema } from 'openai/resources/index'; import { bboxDescription } from './common'; @@ -254,14 +253,17 @@ export const locatorSchema: ResponseFormatJSONSchema = { }, }; -export const findElementPrompt = new PromptTemplate({ - template: ` +export const findElementPrompt = ({ + pageDescription, + targetElementDescription, +}: { + pageDescription: string; + targetElementDescription: string; +}) => ` Here is the item user want to find: ===================================== -{targetElementDescription} +${targetElementDescription} ===================================== -{pageDescription} - `, - inputVariables: ['pageDescription', 'targetElementDescription'], -}); +${pageDescription} + `; diff --git a/packages/core/src/ai-model/prompt/llm-section-locator.ts b/packages/core/src/ai-model/prompt/llm-section-locator.ts index c75041be6..ea3f6540c 100644 --- a/packages/core/src/ai-model/prompt/llm-section-locator.ts +++ b/packages/core/src/ai-model/prompt/llm-section-locator.ts @@ -1,4 +1,3 @@ -import { PromptTemplate } from '@langchain/core/prompts'; import type { TVlModeTypes } from '@midscene/shared/env'; import { bboxDescription } from './common'; @@ -35,11 +34,12 @@ the return value should be like this: `; } -export const sectionLocatorInstruction = new PromptTemplate({ - template: `Here is the target element user interested in: +export const sectionLocatorInstruction = ({ + sectionDescription, +}: { + sectionDescription: string; +}) => `Here is the target element user interested in: -{sectionDescription} +${sectionDescription} - `, - inputVariables: ['sectionDescription'], -}); + `; diff --git a/packages/core/tests/unit-test/prompt/prompt.test.ts b/packages/core/tests/unit-test/prompt/prompt.test.ts index af976f1fe..a9ce26ea0 100644 --- a/packages/core/tests/unit-test/prompt/prompt.test.ts +++ b/packages/core/tests/unit-test/prompt/prompt.test.ts @@ -189,16 +189,16 @@ describe('extract element', () => { expect(prompt).toMatchSnapshot(); }); - it('extract element by extractDataPrompt', async () => { - const prompt = await extractDataQueryPrompt( + it('extract element by extractDataPrompt', () => { + const prompt = extractDataQueryPrompt( 'todo title, string', 'todo title, string', ); expect(prompt).toMatchSnapshot(); }); - it('extract element by extractDataPrompt - object', async () => { - const prompt = await extractDataQueryPrompt('todo title, string', { + it('extract element by extractDataPrompt - object', () => { + const prompt = extractDataQueryPrompt('todo title, string', { foo: 'an array indicates the foo', }); expect(prompt).toMatchSnapshot(); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 63f325f4b..636ef25a0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -656,9 +656,6 @@ importers: '@azure/identity': specifier: 4.5.0 version: 4.5.0 - '@langchain/core': - specifier: 0.3.26 - version: 0.3.26(openai@4.81.0(ws@8.18.3)(zod@3.24.3)) '@midscene/recorder': specifier: workspace:* version: link:../recorder @@ -1493,9 +1490,6 @@ packages: '@bufbuild/protobuf@2.6.0': resolution: {integrity: sha512-6cuonJVNOIL7lTj5zgo/Rc2bKAo4/GvN+rKCrUj7GdEHRzCk8zKOfFwUsL9nAVk5rSIsRmlgcpLzTRysopEeeg==} - '@cfworker/json-schema@4.0.3': - resolution: {integrity: sha512-ZykIcDTVv5UNmKWSTLAs3VukO6NDJkkSKxrgUTDPBkAlORVT3H9n5DbRjRl8xIotklscHdbLIa0b9+y3mQq73g==} - '@changesets/apply-release-plan@6.1.4': resolution: {integrity: sha512-FMpKF1fRlJyCZVYHr3CbinpZZ+6MwvOtWUuO8uo+svcATEoc1zRDcj23pAurJ2TZ/uVz1wFHH6K3NlACy0PLew==} @@ -2618,10 +2612,6 @@ packages: peerDependencies: tslib: '2' - '@langchain/core@0.3.26': - resolution: {integrity: sha512-6RUQHEp8wv+JwtYIIEBYBzbLlcAQZFc7EDOgAM0ukExjh9HiXoJzoWpgMRRCrr/koIbtwXPJUqBprZK1I1CXHQ==} - engines: {node: '>=18'} - '@manypkg/find-root@1.1.0': resolution: {integrity: sha512-mki5uBvhHzO8kYYix/WRy2WX8S3B5wdVSc9D6KcU5lQNglP2yt58/VfLuAK49glRXChosY8ap2oJ1qgma3GUVA==} @@ -5061,10 +5051,6 @@ packages: resolution: {integrity: sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg==} engines: {node: '>=4.0.0'} - commander@10.0.1: - resolution: {integrity: sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==} - engines: {node: '>=14'} - commander@11.1.0: resolution: {integrity: sha512-yPVavfyCcRhmorC7rWlkHn15b4wDVgVmBA7kV4QVBsF7kv/9TKJAbAXVTxvTnwP8HHKjRCJDClKbciiYS7p0DQ==} engines: {node: '>=16'} @@ -6787,9 +6773,6 @@ packages: js-sha256@0.11.0: resolution: {integrity: sha512-6xNlKayMZvds9h1Y1VWc0fQHQ82BxTXizWPEtEeGvmOUYpBRy4gbWroHLpzowe6xiQhHpelCQiE7HEdznyBL9Q==} - js-tiktoken@1.0.16: - resolution: {integrity: sha512-nUVdO5k/M9llWpiaZlBBDdtmr6qWXwSD6fgaDu2zM8UP+OXxx9V37lFkI6w0/1IuaDx7WffZ37oYd9KvcWKElg==} - js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} @@ -6897,14 +6880,6 @@ packages: resolution: {integrity: sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==} engines: {node: '>=6'} - langsmith@0.2.15: - resolution: {integrity: sha512-homtJU41iitqIZVuuLW7iarCzD4f39KcfP9RTBWav9jifhrsDa1Ez89Ejr+4qi72iuBu8Y5xykchsGVgiEZ93w==} - peerDependencies: - openai: '*' - peerDependenciesMeta: - openai: - optional: true - langsmith@0.3.7: resolution: {integrity: sha512-wakN1hxGkm1JR2PpAV7fiT7oC99LKcgxiuUrYGZWPbuj7Y8EPF19F7VNr4B+hA219bfaeWTa4Lxy2YrtPSKnQA==} peerDependencies: @@ -7519,10 +7494,6 @@ packages: resolution: {integrity: sha512-lDmx79y1z6i7RNx0ZGCPq1bzJ6ZoDDKbvh7jxr9SJcWLkShMzXrHbYVpTdnhNM5MXpDUxCQ4DgqVttVXlBgiBQ==} engines: {node: '>=8'} - mustache@4.2.0: - resolution: {integrity: sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==} - hasBin: true - mute-stream@0.0.8: resolution: {integrity: sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==} @@ -10166,11 +10137,6 @@ packages: resolution: {integrity: sha512-zK7YHHz4ZXpW89AHXUPbQVGKI7uvkd3hzusTdotCg1UxyaVtg0zFJSTfW/Dq5f7OBBVnq6cZIaC8Ti4hb6dtCA==} engines: {node: '>= 14'} - zod-to-json-schema@3.24.1: - resolution: {integrity: sha512-3h08nf3Vw3Wl3PK+q3ow/lIil81IT2Oa7YpQyUUDsEWbXveMesdfK1xBd2RhCkynwZndAxixji/7SYJJowr62w==} - peerDependencies: - zod: ^3.24.1 - zod-to-json-schema@3.24.5: resolution: {integrity: sha512-/AuWwMP+YqiPbsJx5D6TfgRTc4kTLjsh5SOcd4bLsfUg2RcEXrFMJl1DGgdHy2aCfsIA/cr/1JM0xcB2GZji8g==} peerDependencies: @@ -10616,8 +10582,6 @@ snapshots: '@bufbuild/protobuf@2.6.0': optional: true - '@cfworker/json-schema@4.0.3': {} - '@changesets/apply-release-plan@6.1.4': dependencies: '@babel/runtime': 7.27.0 @@ -11864,23 +11828,6 @@ snapshots: dependencies: tslib: 2.8.1 - '@langchain/core@0.3.26(openai@4.81.0(ws@8.18.3)(zod@3.24.3))': - dependencies: - '@cfworker/json-schema': 4.0.3 - ansi-styles: 5.2.0 - camelcase: 6.3.0 - decamelize: 1.2.0 - js-tiktoken: 1.0.16 - langsmith: 0.2.15(openai@4.81.0(ws@8.18.3)(zod@3.24.3)) - mustache: 4.2.0 - p-queue: 6.6.2 - p-retry: 4.6.2 - uuid: 10.0.0 - zod: 3.24.3 - zod-to-json-schema: 3.24.1(zod@3.24.3) - transitivePeerDependencies: - - openai - '@manypkg/find-root@1.1.0': dependencies: '@babel/runtime': 7.27.0 @@ -15116,8 +15063,6 @@ snapshots: lodash.camelcase: 4.3.0 typical: 4.0.0 - commander@10.0.1: {} - commander@11.1.0: {} commander@13.1.0: {} @@ -17212,10 +17157,6 @@ snapshots: js-sha256@0.11.0: {} - js-tiktoken@1.0.16: - dependencies: - base64-js: 1.5.1 - js-tokens@4.0.0: {} js-yaml@3.14.1: @@ -17350,17 +17291,6 @@ snapshots: kleur@4.1.5: {} - langsmith@0.2.15(openai@4.81.0(ws@8.18.3)(zod@3.24.3)): - dependencies: - '@types/uuid': 10.0.0 - commander: 10.0.1 - p-queue: 6.6.2 - p-retry: 4.6.2 - semver: 7.7.2 - uuid: 10.0.0 - optionalDependencies: - openai: 4.81.0(ws@8.18.3)(zod@3.24.3) - langsmith@0.3.7(openai@4.81.0(ws@8.18.3)(zod@3.24.3)): dependencies: '@types/uuid': 10.0.0 @@ -18199,8 +18129,6 @@ snapshots: arrify: 2.0.1 minimatch: 3.1.2 - mustache@4.2.0: {} - mute-stream@0.0.8: {} mute-stream@2.0.0: {} @@ -21399,10 +21327,6 @@ snapshots: compress-commons: 6.0.2 readable-stream: 4.7.0 - zod-to-json-schema@3.24.1(zod@3.24.3): - dependencies: - zod: 3.24.3 - zod-to-json-schema@3.24.5(zod@3.24.3): dependencies: zod: 3.24.3