Merge branch 'main' into dev/robgruen/electron_tests

pipelines/azure-build-ts.yml

@@ -38,16 +38,16 @@ jobs:
         displayName: "Checkout TypeAgent Repository"
         path: "typeagent"
 
-      - template: include-install-pnpm.yml
-        parameters:
-          buildDirectory: $(Build.SourcesDirectory)/ts
-
       - task: UseNode@1
         displayName: "Setup Node.js"
         inputs:
           version: $(nodeVersion)
           checkLatest: true
 
+      - template: include-install-pnpm.yml
+        parameters:
+          buildDirectory: $(Build.SourcesDirectory)/ts
+
       - script: |
           pnpm install --frozen-lockfile --strict-peer-dependencies
         displayName: "Install dependencies"

pipelines/include-install-pnpm.yml

@@ -45,6 +45,7 @@ steps:
       # workspace-concurrency 0 means use use the CPU core count. This is better than the default (4) for larger agents.
       script: |
         echo "Using node $(node --version)"
+        npm install -g [email protected]
         sudo corepack enable
         echo "Using pnpm $(pnpm -v)"
         pnpm config set store-dir ${{ parameters.pnpmStorePath }}

ts/Dockerfile

@@ -2,6 +2,7 @@ FROM node:20 AS base
 
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
+RUN npm install -g [email protected]
 RUN corepack enable
 
 # Install dependencies required for Chrome/Puppeteer

ts/package.json

@@ -48,7 +48,7 @@
     "prettier": "^3.2.5",
     "shx": "^0.3.4"
   },
-  "packageManager": "[email protected].4+sha512.b2dc20e2fc72b3e18848459b37359a32064663e5627a51e4c74b2c29dd8e8e0491483c3abb40789cfd578bf362fb6ba8261b05f0387d76792ed6e23ea3b1b6a0",
+  "packageManager": "[email protected].5+sha512.845196026aab1cc3f098a0474b64dfbab2afe7a1b4e91dd86895d8e4aa32a7a6d03049e2d0ad770bbe4de023a7122fb68c1a1d6e0d033c7076085f9d5d4800d4",
   "engines": {
     "node": ">=18",
     "pnpm": ">=9"

ts/packages/agents/browser/src/agent/discovery/actionHandler.mts

@@ -24,16 +24,36 @@ export async function handleSchemaDiscoveryAction(
     case "findUserActions":
       await handleFindUserActions(action);
       break;
+    case "summarizePage":
+      await handleGetPageSummary(action);
+      break;
   }
 
   async function handleFindUserActions(action: any) {
     const htmlFragments = await browser.getHtmlFragments();
+    // const screenshot = await browser.getCurrentPageScreenshot();
+    const screenshot = "";
+    let pageSummary = "";
+
+    const summaryResponse = await agent.getPageSummary(
+      undefined,
+      htmlFragments,
+      screenshot,
+    );
+
+    if (summaryResponse.success) {
+      pageSummary =
+        "Page summary: \n" + JSON.stringify(summaryResponse.data, null, 2);
+    }
+
     const timerName = `Analyzing page actions`;
     console.time(timerName);
+
     const response = await agent.getCandidateUserActions(
       undefined,
       htmlFragments,
-      undefined,
+      screenshot,
+      pageSummary,
     );
 
     if (!response.success) {
@@ -48,5 +68,22 @@ export async function handleSchemaDiscoveryAction(
     return response.data;
   }
 
+  async function handleGetPageSummary(action: any) {
+    const htmlFragments = await browser.getHtmlFragments();
+    const timerName = `Summarizing page`;
+    console.time(timerName);
+    const response = await agent.getPageSummary(undefined, htmlFragments);
+
+    if (!response.success) {
+      console.error("Attempt to get page summary failed");
+      console.error(response.message);
+      return;
+    }
+
+    console.timeEnd(timerName);
+    message = "Page summary: \n" + JSON.stringify(response.data, null, 2);
+    return response.data;
+  }
+
   return message;
 }

ts/packages/agents/browser/src/agent/discovery/schema/discoveryActions.mts

@@ -14,6 +14,9 @@ export type FindUserActions = {
 
 export type SummarizePage = {
   actionName: "summarizePage";
+  parameters: {
+    allowDuplicates?: boolean;
+  };
 };
 
 export type SaveUserActions = {

ts/packages/agents/browser/src/agent/discovery/schema/pageSummary.mts

@@ -0,0 +1,10 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+// A description of the page, including layout information and summary of content.
+export type PageDescription = {
+  description: string;
+  features: string[];
+  entities: string[];
+  possibleUserAction: string[];
+};

ts/packages/agents/browser/src/agent/discovery/schema/pageTypes.mts

@@ -1,55 +1,71 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-export type SearchBox = {
-  featureName: "searchInputBox";
-  description: "Input box for searching on the page";
-  parameters: {
-    cssSelector: string;
-  };
-};
-
-export type SearchResultsList = {
-  featureName: "searchResultsList";
-  description: "List of products available from the search results";
-  parameters: {
-    cssSelector: string;
-  };
-};
-
-export type ProductDetailsCard = {
-  featureName: "productDetailsCard";
-  description: "A section that shows the product name, price, images and rating. This also gives an option to add the product to the shopping cart.";
-  parameters: {
-    cssSelector: string;
-  };
-};
-
-export type SearchForContent = {
-  actionName: "searchForProduct";
-  description: "Find content on the page";
-  parameters: {
-    value: string;
-    cssSelector: string;
-  };
-};
-
 export type LandingPage = {
   description: "The default landing page for the site";
-  features: SearchBox;
 };
 
 export type SearchResultsPage = {
   description: "The search results page";
-  features: SearchResultsList;
 };
 
 export type ProductDetailsPage = {
   description: "A product details page, with focus on one product.";
-  features: ProductDetailsCard;
 };
 
 export type ShoppingCartPage = {
   description: "The shopping cart page for the site";
-  features: SearchBox;
 };
+
+export type PastOrderPage = {
+  description: "The page showing a user's past orders";
+};
+
+export type UnknownPage = {
+  description: "A page that does not meet the previous more-specific categories";
+};
+
+export type CommercePageTypes =
+  | LandingPage
+  | SearchResultsPage
+  | ProductDetailsPage
+  | ShoppingCartPage
+  | PastOrderPage
+  | UnknownPage;
+
+export type CrosswordPage = {
+  description: "The page showing a crossword puzzle";
+};
+
+export type NewsLandingPage = {
+  description: "The page showing news headlines for the day";
+};
+
+export type SportsLandingPage = {
+  description: "The page showing sports headlines for the day";
+};
+
+export type OpinionPage = {
+  description: "The page showing editorial opinions for the day";
+};
+
+export type ArticlePage = {
+  description: "The page showing an individual news article";
+};
+
+export type WeatherPage = {
+  description: "The page showing weather headlines";
+};
+
+export type PuzzlesPage = {
+  description: "The page showing a list of puzzles, such as sudoku, crossword, word matching games and more.";
+};
+
+export type NewsPageTypes =
+  | CrosswordPage
+  | NewsLandingPage
+  | SportsLandingPage
+  | OpinionPage
+  | ArticlePage
+  | PuzzlesPage
+  | UnknownPage;

ts/packages/agents/browser/src/agent/discovery/schema/userActionsPool.mts

@@ -28,6 +28,7 @@ export type SearchForProductAction = {
   };
 };
 
+// This allows users to select individual results on the search results page.
 export type SelectSearchResult = {
   actionName: "selectSearchResult";
   parameters: {
@@ -38,39 +39,63 @@ export type SelectSearchResult = {
 
 export type NavigateToHomePage = {
   actionName: "navigateToHomePage";
+  parameters: {
+    linkCssSelector: string;
+  };
 };
 
 // Follow a link to view  a store landing page
 export type NavigateToStorePage = {
   actionName: "navigateToStorePage";
+  parameters: {
+    linkCssSelector: string;
+  };
 };
 
 // Follow a link to view  a product details page
 export type NavigateToProductPage = {
   actionName: "navigateToProductPage";
+  parameters: {
+    linkCssSelector: string;
+  };
 };
 
-// Follow a link to view  a recipe details page
+// Follow a link to view  a recipe details page. This link is typically named "Recipe" or "Recipes"
 export type NavigateToRecipePage = {
   actionName: "navigateToRecipePage";
+  parameters: {
+    linkCssSelector: string;
+  };
 };
 
 export type NavigateToListPage = {
   actionName: "navigateToListPage";
+  parameters: {
+    linkCssSelector: string;
+  };
 };
 
+// Navigate to the "Buy it again" page. This page may also be called Past Orders.
 export type NavigateToBuyItAgainPage = {
   actionName: "navigateToBuyItAgainPage";
+  parameters: {
+    linkCssSelector: string;
+  };
 };
 
+// This link opens the shopping cart. Its usually indicated by a cart or bag icon.
 export type NavigateToShoppingCartPage = {
   actionName: "navigateToShoppingCartPage";
+  parameters: {
+    linkCssSelector: string;
+  };
 };
 
 export type NavigateToOtherPage = {
   actionName: "navigateToOtherPage";
   parameters: {
     pageType: string;
+    linkCssSelector: string;
   };
 };