Skip to content

Commit

Permalink
Bug fixes for commerce scenarios (#540)
Browse files Browse the repository at this point in the history
  • Loading branch information
@hillary-mutisya
hillary-mutisya authored Jan 9, 2025
1 parent fd2a9cf commit 4a9baa4
Show file tree
Hide file tree
Showing 5 changed files with 93 additions and 94 deletions.
166 changes: 85 additions & 81 deletions ts/packages/agents/browser/src/agent/instacart/actionHandler.mts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
ListDetailsInfo,
ListInfo,
ListsNavigationLink,
NearbyStoresList,
ProductDetailsHeroTile,
ProductTile,
RecipeHeroSection,
Expand All @@ -37,10 +38,7 @@ export async function handleInstacartAction(

switch (action.actionName) {
case "searchForProductAction":
await searchForProduct(action.parameters.keyword);
break;
case "selectSearchResult":
await selectSearchResult(action.parameters.productName);
await handleFindProduct(action);
break;
case "addToCartAction":
await handleAddToCart(action);
Expand All @@ -66,10 +64,10 @@ export async function handleInstacartAction(
break;
}

async function getComponentFromPage(
async function getPageComponent<T>(
componentType: string,
selectionCondition?: string,
) {
): Promise<T | undefined> {
const htmlFragments = await browser.getHtmlFragments(true);
const timerName = `getting ${componentType} section`;

Expand All @@ -82,17 +80,26 @@ export async function handleInstacartAction(
);

if (!response.success) {
console.error("Attempt to get product tilefailed");
console.error("Attempt to get page component failed");
console.error(response.message);
return;
return undefined;
}

console.timeEnd(timerName);
return response.data;
return response.data as T;
}

async function handleFindProduct(action: any) {
await searchForProduct(action.parameters.keyword);
await selectProductSearchResult(action.parameters.keyword);
}

async function searchForProduct(productName: string) {
const selector = (await getComponentFromPage("SearchInput")) as SearchInput;
const selector = await getPageComponent<SearchInput>("SearchInput");
if (!selector) {
return;
}

const searchSelector = selector.cssSelector;

await browser.clickOn(searchSelector);
Expand All @@ -102,55 +109,56 @@ export async function handleInstacartAction(
await browser.awaitPageLoad();
}

async function selectSearchResult(productName: string) {
async function selectProductSearchResult(productName: string) {
const request = `Search result: ${productName}`;
const targetProduct = (await getComponentFromPage(
const targetProduct = await getPageComponent<ProductTile>(
"ProductTile",
request,
)) as ProductTile;

await browser.clickOn(targetProduct.detailsLinkSelector);
await browser.awaitPageInteraction();
await browser.awaitPageLoad();
);
if (targetProduct?.detailsLinkSelector) {
await browser.clickOn(targetProduct.detailsLinkSelector);
await browser.awaitPageInteraction();
await browser.awaitPageLoad();
}
}

async function handleAddToCart(action: any) {
const targetProduct = (await getComponentFromPage(
const targetProduct = await getPageComponent<ProductDetailsHeroTile>(
"ProductDetailsHeroTile",
)) as ProductDetailsHeroTile;
);

if (targetProduct.addToCartButton) {
if (targetProduct?.addToCartButton) {
await browser.clickOn(targetProduct.addToCartButton.cssSelector);
}
}

async function handleAddToList(action: any) {
const targetProduct = (await getComponentFromPage(
const targetProduct = await getPageComponent<ProductDetailsHeroTile>(
"ProductDetailsHeroTile",
)) as ProductDetailsHeroTile;
);

if (targetProduct.addToListButton) {
if (targetProduct?.addToListButton) {
await browser.clickOn(targetProduct.addToListButton.cssSelector);

// this launches a popup with the available lists
const request = `ListName: ${action.listName}`;
const targetList = (await getComponentFromPage(
const targetList = await getPageComponent<AllListsInfo>(
"AllListsInfo",
request,
)) as AllListsInfo;
);

if (targetList) {
if (targetList?.lists) {
await browser.clickOn(targetList.lists[0].cssSelector);
await browser.clickOn(targetList.submitButtonCssSelector);
}
}
}

async function goToHomepage() {
const link = (await getComponentFromPage("HomeLink")) as HomeLink;
const link = await getPageComponent<HomeLink>("HomeLink");
console.log(link);

if (link.linkCssSelector) {
if (link?.linkCssSelector) {
await browser.clickOn(link.linkCssSelector);
await browser.awaitPageInteraction();
await browser.awaitPageLoad(5000);
Expand All @@ -159,33 +167,36 @@ export async function handleInstacartAction(

async function handleFindStores(action: any) {
await goToHomepage();
const stores = (await getComponentFromPage("StoreInfo")) as StoreInfo[];
console.log(stores);
return stores;
const storesList =
await getPageComponent<NearbyStoresList>("NearbyStoresList");
console.log(storesList);
return storesList;
}

async function searchForStore(storeName: string) {
await goToHomepage();
const selector = (await getComponentFromPage("SearchInput")) as SearchInput;
const searchSelector = selector.cssSelector;
const selector = await getPageComponent<SearchInput>("SearchInput");

await browser.clickOn(searchSelector);
await browser.enterTextIn("store: " + storeName, searchSelector);
await browser.clickOn(selector.submitButtonCssSelector);
await browser.awaitPageInteraction();
await browser.awaitPageLoad();
if (selector?.cssSelector && selector?.submitButtonCssSelector) {
const searchSelector = selector.cssSelector;

await browser.clickOn(searchSelector);
await browser.enterTextIn("store: " + storeName, searchSelector);
await browser.clickOn(selector.submitButtonCssSelector);
await browser.awaitPageInteraction();
await browser.awaitPageLoad();
}
}

async function selectStoreSearchResult(storeName: string) {
const request = `${storeName}`;
const targetStore = (await getComponentFromPage(
"StoreInfo",
request,
)) as StoreInfo;
const targetStore = await getPageComponent<StoreInfo>("StoreInfo", request);

await browser.clickOn(targetStore.storeLinkCssSelector);
await browser.awaitPageInteraction(1000);
await browser.awaitPageLoad(5000);
if (targetStore?.storeLinkCssSelector) {
await browser.clickOn(targetStore.storeLinkCssSelector);
await browser.awaitPageInteraction(1000);
await browser.awaitPageLoad(5000);
}
}

async function handleSetPreferredStore(action: any) {
Expand All @@ -197,7 +208,10 @@ export async function handleInstacartAction(

async function searchForRecipe(recipeKeywords: string) {
await goToHomepage();
const selector = (await getComponentFromPage("SearchInput")) as SearchInput;
const selector = await getPageComponent<SearchInput>("SearchInput");
if (!selector) {
return;
}
const searchSelector = selector.cssSelector;

await browser.clickOn(searchSelector);
Expand All @@ -209,13 +223,13 @@ export async function handleInstacartAction(

async function selectRecipeSearchResult(recipeKeywords: string) {
const request = `${recipeKeywords}`;
const allRecipes = (await getComponentFromPage(
const allRecipes = await getPageComponent<AllRecipeSearchResults>(
"AllRecipeSearchResults",
request,
)) as AllRecipeSearchResults;
);

console.log(allRecipes);
if (allRecipes.recipes.length > 0) {
if (allRecipes && allRecipes.recipes.length > 0) {
await browser.clickOn(allRecipes.recipes[0].recipeLinkCssSelector);
await browser.awaitPageInteraction();
console.log(
Expand All @@ -235,38 +249,33 @@ export async function handleInstacartAction(
await searchForRecipe(action.parameters.keywords);
await selectRecipeSearchResult(action.parameters.keywords);

const targetRecipe = (await getComponentFromPage(
"RecipeHeroSection",
)) as RecipeHeroSection;
const targetRecipe =
await getPageComponent<RecipeHeroSection>("RecipeHeroSection");

if (targetRecipe && targetRecipe.addAllIngridientsCssSelector) {
if (targetRecipe?.addAllIngridientsCssSelector) {
await browser.clickOn(targetRecipe.addAllIngridientsCssSelector);
}
}

async function handleBuyListContents(action: any) {
const navigationLink = (await getComponentFromPage(
const navigationLink = await getPageComponent<ListsNavigationLink>(
"ListsNavigationLink",
)) as ListsNavigationLink;
);

if (navigationLink) {
if (navigationLink?.linkCssSelector) {
await browser.clickOn(navigationLink.linkCssSelector);
await browser.awaitPageInteraction();
await browser.awaitPageLoad();

const request = `List name: ${action.parameters.listName}`;
const targetList = (await getComponentFromPage(
"ListInfo",
request,
)) as ListInfo;
const targetList = await getPageComponent<ListInfo>("ListInfo", request);

if (targetList && targetList.detailsLinkCssSelector) {
if (targetList?.detailsLinkCssSelector) {
await browser.clickOn(targetList.detailsLinkCssSelector);
await browser.awaitPageLoad();

const listDetails = (await getComponentFromPage(
"ListDetailsInfo",
)) as ListDetailsInfo;
const listDetails =
await getPageComponent<ListDetailsInfo>("ListDetailsInfo");

if (listDetails && listDetails.products) {
for (let product of listDetails.products) {
Expand All @@ -282,28 +291,23 @@ export async function handleInstacartAction(
async function selectStore(storeName: string) {
await goToHomepage();
const request = `Store name: ${storeName}`;
const targetStore = (await getComponentFromPage(
"StoreInfo",
request,
)) as StoreInfo;
const targetStore = await getPageComponent<StoreInfo>("StoreInfo", request);

console.log(targetStore);

if (!targetStore) {
return;
if (targetStore?.storeLinkCssSelector) {
await browser.clickOn(targetStore.storeLinkCssSelector);
await browser.awaitPageInteraction();
await browser.awaitPageLoad();
}

await browser.clickOn(targetStore.storeLinkCssSelector);
await browser.awaitPageInteraction();
await browser.awaitPageLoad();
}

async function handleBuyItAgain(action: any) {
await selectStore(action.parameters.storeName);

const navigationLink = (await getComponentFromPage(
const navigationLink = await getPageComponent<BuyItAgainNavigationLink>(
"BuyItAgainNavigationLink",
)) as BuyItAgainNavigationLink;
);

console.log(navigationLink);

Expand All @@ -312,12 +316,12 @@ export async function handleInstacartAction(
await browser.awaitPageInteraction();
await browser.awaitPageLoad();

const headerSection = (await getComponentFromPage(
const headerSection = await getPageComponent<BuyItAgainHeaderSection>(
"BuyItAgainHeaderSection",
)) as BuyItAgainHeaderSection;
);
console.log(headerSection);

if (headerSection && headerSection.products) {
if (headerSection?.products) {
if (action.parameters.allItems) {
for (let product of headerSection.products) {
if (product.addToCartButtonCssSelector) {
Expand All @@ -327,10 +331,10 @@ export async function handleInstacartAction(
}
} else {
const request = `Product: ${action.productName}`;
const targetProduct = (await getComponentFromPage(
const targetProduct = await getPageComponent<ProductTile>(
"ProductTile",
request,
)) as ProductTile;
);
if (targetProduct && targetProduct.addToCartButtonCssSelector) {
await browser.clickOn(targetProduct.addToCartButtonCssSelector);
await browser.awaitPageInteraction();
Expand Down
4 changes: 4 additions & 0 deletions ts/packages/agents/browser/src/agent/instacart/schema/pageComponents.mts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,10 @@ export type StoreInfo = {
storeLinkCssSelector: string;
};

export type NearbyStoresList = {
stores: StoreInfo[];
};

export type RecipeBuyButton = {
cssSelector: string;
label: string;
Expand Down
11 changes: 1 addition & 10 deletions ts/packages/agents/browser/src/agent/instacart/schema/userActions.mts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,14 +100,6 @@ export type SearchForRecipeAction = {
};
};

export type SelectSearchResult = {
actionName: "selectSearchResult";
parameters: {
position?: number;
productName?: string;
};
};

export type InstacartActions =
| AddToCartAction
| AddToListAction
Expand All @@ -122,5 +114,4 @@ export type InstacartActions =
| SaveRecipeAction
| DeleteRecipeAction
| SearchForProductAction
| SearchForRecipeAction
| SelectSearchResult;
| SearchForRecipeAction;
4 changes: 2 additions & 2 deletions ts/packages/agents/browser/src/electron/manifest.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,13 @@
"matches": ["https://*/*"],
"js": ["contentScript.js"],
"all_frames": true,
"runAt": "document_start"
"run_at": "document_start"
},
{
"matches": ["https://*/*"],
"js": ["agentActivation.js"],
"world": "MAIN",
"runAt": "document_start"
"run_at": "document_start"
},
{
"matches": ["https://*/*"],
Expand Down
2 changes: 1 addition & 1 deletion ts/packages/agents/browser/src/extension/manifest.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
{
"matches": ["https://*/*"],
"js": ["contentScript.js"],
"runAt": "document_start",
"run_at": "document_start",
"all_frames": true
},
{
Expand Down

0 comments on commit 4a9baa4

Please sign in to comment.