Merge branch 'main' into dev/pialic/docupro

microsoft · Feb 4, 2025 · 1be54dc · 1be54dc
2 parents 6f3903b + 7901207
commit 1be54dc
Show file tree

Hide file tree

Showing 35 changed files with 661 additions and 321 deletions.
diff --git a/.github/workflows/build-ts.yml b/.github/workflows/build-ts.yml
@@ -66,10 +66,6 @@ jobs:
         working-directory: ts
         run: |
           pnpm install --frozen-lockfile --strict-peer-dependencies
-      # - name: Install Playwright Browsers
-      #   if: ${{ runner.os == 'windows' && matrix.version == '22' }}
-      #   run: pnpm exec playwright install --with-deps
-      #   working-directory: ts/packages/shell
       - name: Build
         if: ${{ github.event_name != 'pull_request' || steps.filter.outputs.ts == 'true' }}
         working-directory: ts
@@ -85,43 +81,3 @@ jobs:
         working-directory: ts
         run: |
           npm run lint
-      # - name: Login to Azure
-      #   if: ${{ github.event_name != 'merge_group' }}
-      #   uses: azure/[email protected]
-      #   with:
-      #     client-id: ${{ secrets.AZUREAPPSERVICE_CLIENTID_5B0D2D6BA40F4710B45721D2112356DD }}
-      #     tenant-id: ${{ secrets.AZUREAPPSERVICE_TENANTID_39BB903136F14B6EAD8F53A8AB78E3AA }}
-      #     subscription-id: ${{ secrets.AZUREAPPSERVICE_SUBSCRIPTIONID_F36C1F2C4B2C49CA8DD5C52FAB98FA30 }}
-      # - name: Get Keys
-      #   if: ${{ github.event_name != 'merge_group' }}
-      #   run: |
-      #     node tools/scripts/getKeys.mjs --vault build-pipeline-kv
-      #   working-directory: ts
-      # - name: Test CLI - smoke
-      #   if: ${{ github.event_name != 'merge_group' }}
-      #   run: |
-      #     npm run start:dev 'prompt' 'why is the sky blue'
-      #   working-directory: ts/packages/cli
-      #   continue-on-error: true
-      # - name: Shell Tests - smoke (windows)
-      #   if: ${{ github.event_name != 'merge_group' && runner.os == 'windows' && matrix.version == '22' }}
-      #   timeout-minutes: 60
-      #   run: |
-      #     npx playwright test simple.spec.ts
-      #     rm ../../.env
-      # - name: Shell Tests - smoke (linux)
-      #   if: ${{ github.event_name != 'merge_group' && runner.os == 'Linux' && matrix.version == '22' }}
-      #   timeout-minutes: 60
-      #   run: |
-      #     Xvfb :99 -screen 0 1600x1200x24 & export DISPLAY=:99
-      #     npx playwright test simple.spec.ts
-      #     rm ../../.env          
-      #   working-directory: ts/packages/shell
-      #   continue-on-error: true
-      # - name: Live Tests
-      #   if: ${{ github.event_name != 'merge_group' && runner.os == 'linux' && matrix.version == '22' }}
-      #   timeout-minutes: 60
-      #   run: |
-      #     npm run test:live
-      #   working-directory: ts
-      #   continue-on-error: true
diff --git a/.github/workflows/shell-tests.yml b/.github/workflows/shell-tests.yml
@@ -28,7 +28,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["windows-latest"]
+        os: ["windows-latest", "ubuntu-latest"]
         version: [20]
 
     runs-on: ${{ matrix.os }}
@@ -98,19 +98,15 @@ jobs:
         timeout-minutes: 60
         run: |
           npm run shell:test
-          rm ../../.env
         working-directory: ts/packages/shell
         continue-on-error: true
 
       - name: Shell Tests (linux)
         if: ${{ runner.os == 'Linux' }}
         timeout-minutes: 60
-        # https://github.com/microsoft/playwright/issues/34251 - sysctl command
         run: |
-          sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 
           Xvfb :99 -screen 0 1600x1200x24 & export DISPLAY=:99
           npm run shell:test
-          rm ../../.env
         working-directory: ts/packages/shell
         continue-on-error: true
 
@@ -129,3 +125,9 @@ jobs:
           path: ts/packages/shell/playwright-report/
           overwrite: true
           retention-days: 30
+
+      - name: Clean up Keys
+        run: |
+          rm ./.env
+        working-directory: ts
+        if: always()          
diff --git a/.github/workflows/smoke-tests-pull_request_targets.yml.bak b/.github/workflows/smoke-tests-pull_request_targets.yml.bak
diff --git a/.github/workflows/smoke-tests.yml b/.github/workflows/smoke-tests.yml
@@ -9,8 +9,6 @@ on:
   workflow_dispatch:
   push:
     branches: ["main"]
-  # pull_request:
-  #   branches: ["main"]
   pull_request_target:
     branches: ["main"]
   merge_group:
@@ -27,7 +25,9 @@ permissions:
 
 env:
   NODE_OPTIONS: --max_old_space_size=8192
-  DEBUG: pw:browser*
+  #  DEBUG: pw:browser*                       # PlayWright debug messages
+  #  ELECTRON_ENABLE_LOGGING: true            # Electron debug messages
+  #  DEBUG: typeagent:*                       # TypeAgent debug messages
 
 jobs:
   shell_and_cli:
@@ -37,7 +37,6 @@ jobs:
       fail-fast: false
       matrix:
         os: ["ubuntu-latest", "windows-latest"]
-        #os: ["ubuntu-latest"]
         version: [20]
 
     runs-on: ${{ matrix.os }}

diff --git a/ts/packages/agents/browser/src/agent/browserConnector.mts b/ts/packages/agents/browser/src/agent/browserConnector.mts
@@ -80,13 +80,17 @@ export class BrowserConnector {
     return [];
   }
 
-  async getFilteredHtmlFragments(inputHtmlFragments: any[]) {
+  async getFilteredHtmlFragments(
+    inputHtmlFragments: any[],
+    cssSelectorsToKeep: string[],
+  ) {
     let htmlFragments: any[] = [];
     const timeoutPromise = new Promise((f) => setTimeout(f, 5000));
     const filterAction = {
       actionName: "getFilteredHTMLFragments",
       parameters: {
         fragments: inputHtmlFragments,
+        cssSelectorsToKeep: cssSelectorsToKeep,
       },
     };
 

diff --git a/ts/packages/agents/browser/src/agent/discovery/actionHandler.mts b/ts/packages/agents/browser/src/agent/discovery/actionHandler.mts
@@ -24,16 +24,39 @@ export async function handleSchemaDiscoveryAction(
     case "findUserActions":
       await handleFindUserActions(action);
       break;
+    case "summarizePage":
+      await handleGetPageSummary(action);
+      break;
+    case "findPageComponents":
+      await handleGetPageComponents(action);
+      break;
   }
 
   async function handleFindUserActions(action: any) {
     const htmlFragments = await browser.getHtmlFragments();
+    // const screenshot = await browser.getCurrentPageScreenshot();
+    const screenshot = "";
+    let pageSummary = "";
+
+    const summaryResponse = await agent.getPageSummary(
+      undefined,
+      htmlFragments,
+      screenshot,
+    );
+
+    if (summaryResponse.success) {
+      pageSummary =
+        "Page summary: \n" + JSON.stringify(summaryResponse.data, null, 2);
+    }
+
     const timerName = `Analyzing page actions`;
     console.time(timerName);
+
     const response = await agent.getCandidateUserActions(
       undefined,
       htmlFragments,
-      undefined,
+      screenshot,
+      pageSummary,
     );
 
     if (!response.success) {
@@ -48,5 +71,40 @@ export async function handleSchemaDiscoveryAction(
     return response.data;
   }
 
+  async function handleGetPageSummary(action: any) {
+    const htmlFragments = await browser.getHtmlFragments();
+    const timerName = `Summarizing page`;
+    console.time(timerName);
+    const response = await agent.getPageSummary(undefined, htmlFragments);
+
+    if (!response.success) {
+      console.error("Attempt to get page summary failed");
+      console.error(response.message);
+      return;
+    }
+
+    console.timeEnd(timerName);
+    message = "Page summary: \n" + JSON.stringify(response.data, null, 2);
+    return response.data;
+  }
+
+  async function handleGetPageComponents(action: any) {
+    const htmlFragments = await browser.getHtmlFragments();
+    const timerName = `Getting page layout`;
+    console.time(timerName);
+    const response = await agent.getPageLayout(undefined, htmlFragments);
+
+    if (!response.success) {
+      console.error("Attempt to get page layout failed");
+      console.error(response.message);
+      return;
+    }
+
+    console.timeEnd(timerName);
+    message = "Page layout: \n" + JSON.stringify(response.data, null, 2);
+
+    return response.data;
+  }
+
   return message;
 }
diff --git a/ts/packages/agents/browser/src/agent/discovery/schema/discoveryActions.mts b/ts/packages/agents/browser/src/agent/discovery/schema/discoveryActions.mts
@@ -7,9 +7,6 @@ export type FindPageComponents = {
 
 export type FindUserActions = {
   actionName: "findUserActions";
-  parameters: {
-    allowDuplicates?: boolean;
-  };
 };
 
 export type SummarizePage = {

diff --git a/ts/packages/agents/browser/src/agent/discovery/schema/pageLayout.mts b/ts/packages/agents/browser/src/agent/discovery/schema/pageLayout.mts
@@ -0,0 +1,9 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+export type PageLayout = {
+  headerCSSSelector: string;
+  footerCSSSelector: string;
+  navigationLinksCSSSelector: string;
+  mainContentCSSSelector: string;
+};
diff --git a/ts/packages/agents/browser/src/agent/discovery/schema/pageSummary.mts b/ts/packages/agents/browser/src/agent/discovery/schema/pageSummary.mts
@@ -0,0 +1,10 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+// A description of the page, including layout information and summary of content.
+export type PageDescription = {
+  description: string;
+  features: string[];
+  entities: string[];
+  possibleUserAction: string[];
+};