Improving find-in-store scenario robustness (#479)

microsoft · Dec 11, 2024 · cffe06e · cffe06e
1 parent a53f8b8
commit cffe06e
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 17 deletions.
diff --git a/ts/packages/agents/browser/src/agent/commerce/actionHandler.mts b/ts/packages/agents/browser/src/agent/commerce/actionHandler.mts
@@ -79,18 +79,18 @@ export async function handleCommerceAction(
     await browser.clickOn(searchSelector);
     await browser.enterTextIn(productName, searchSelector);
     await browser.clickOn(selector.submitButtonCssSelector);
-    await new Promise((r) => setTimeout(r, 200));
+    await new Promise((r) => setTimeout(r, 400));
     await browser.awaitPageLoad();
   }
 
   async function selectSearchResult(productName: string) {
     const request = `Search result: ${productName}`;
-    const selector = (await getComponentFromPage(
+    const targetProduct = (await getComponentFromPage(
       "ProductTile",
       request,
     )) as ProductTile;
-    console.log(selector);
-    await browser.clickOn(selector.detailsLinkSelector);
+
+    await browser.clickOn(targetProduct.detailsLinkSelector);
     await new Promise((r) => setTimeout(r, 200));
     await browser.awaitPageLoad();
   }
@@ -121,14 +121,14 @@ export async function handleCommerceAction(
   async function handleFindInStore(action: any) {
     await searchForProduct(action.parameters.productName);
     await selectSearchResult(action.parameters.productName);
-    console.log("Selected search result");
+
+    // wait for delay-loaded items to settle aeven after pageLoad is declared
+    await new Promise((r) => setTimeout(r, 1000));
 
     const targetProduct = (await getComponentFromPage(
       "ProductDetailsHeroTile",
     )) as ProductDetailsHeroTile;
 
-    console.log(targetProduct);
-
     if (targetProduct && targetProduct.physicalLocationInStore) {
       message = `Found ${targetProduct.numberInStock} at ${targetProduct.physicalLocationInStore} in the ${targetProduct.storeName} store`;
       return;

diff --git a/ts/packages/agents/browser/src/agent/commerce/schema/pageComponents.mts b/ts/packages/agents/browser/src/agent/commerce/schema/pageComponents.mts
@@ -1,11 +1,6 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-export type AddToCartButton = {
-  // css selector for the add to cart button
-  cssSelector: string;
-};
-
 export type ProductTile = {
   productName: string;
   price: string;
@@ -25,7 +20,11 @@ export type ProductDetailsHeroTile = {
   // css selector for text input
   cssSelector: string;
 
-  addToCartButton?: AddToCartButton;
+  addToCartButton?: {
+    // css selector for the add to cart button
+    cssSelector: string;
+  };
+
   storeName?: string;
   // The physical location of the goods, such as the Aisle, Bay or Shelf
   // Only provide this infomration if it present on the page

diff --git a/ts/packages/agents/browser/src/agent/commerce/schema/userActions.mts b/ts/packages/agents/browser/src/agent/commerce/schema/userActions.mts
@@ -10,10 +10,6 @@ export type AddToCartAction = {
 
 export type FindNearbyStoreAction = {
   actionName: "findNearbyStoreAction";
-  parameters: {
-    productName: string;
-    storeLocation?: string;
-  };
 };
 
 // Use this action for user queries such as "where is product X in the store"

diff --git a/ts/packages/shell/demo/headlessBrowser.txt b/ts/packages/shell/demo/headlessBrowser.txt
@@ -0,0 +1,7 @@
+@browser launch hidden
+
+open homedepot.com
+
+get the location of aaa batteries in the store
+get the location of step ladders in the store
+get the location of table saws in the store