Improve crossword schema initialization (#492)

microsoft · Dec 13, 2024 · f790c63 · f790c63
1 parent 7a8c027
commit f790c63
Show file tree

Hide file tree

Showing 5 changed files with 36 additions and 28 deletions.
diff --git a/ts/packages/agents/browser/src/agent/browserConnector.mts b/ts/packages/agents/browser/src/agent/browserConnector.mts
@@ -78,6 +78,7 @@ export class BrowserConnector {
       parameters: {
         fullHTML: false,
         downloadAsFile: false,
+        extractText: false,
       },
     };
 

diff --git a/ts/packages/agents/browser/src/agent/crossword/actionHandler.mts b/ts/packages/agents/browser/src/agent/crossword/actionHandler.mts
@@ -31,12 +31,7 @@ export async function getBoardSchema(
 
     for (let i = 0; i < htmlFragments.length; i++) {
       // skip html fragments that are too short to contain crossword
-      if (
-        !htmlFragments[i].content ||
-        htmlFragments[i].content.length < 500 ||
-        !htmlFragments[i].text ||
-        htmlFragments[i].text.length < 200
-      ) {
+      if (!htmlFragments[i].content || htmlFragments[i].content.length < 500) {
         continue;
       }
 
@@ -128,7 +123,7 @@ export async function handleCrosswordAction(
 
       if (selector) {
         await browser.clickOn(selector);
-        await browser.enterTextIn(text);
+        await browser.enterTextIn(text.replace(/\s/g, "")?.toUpperCase());
         message = `OK. Setting the value of ${number} ${direction} to "${text}"`;
       } else {
         message = `${number} ${direction} is not a valid position for this crossword`;

diff --git a/ts/packages/agents/browser/src/agent/crossword/translator.mts b/ts/packages/agents/browser/src/agent/crossword/translator.mts
@@ -216,8 +216,8 @@ export class CrosswordPageTranslator<T extends object> {
   }
 
   getIsCrosswordPresentPromptSections(fragments?: HtmlFragments[]) {
-    // const htmlSection = getHtmlPromptSection(fragments);
-    const htmlSection = getHtmlTextOnlyPromptSection(fragments);
+    const htmlSection = getHtmlPromptSection(fragments);
+    // const htmlSection = getHtmlTextOnlyPromptSection(fragments);
     const prefixSection = getBootstrapPrefixPromptSection();
     const promptSections = [
       ...prefixSection,

diff --git a/ts/packages/agents/browser/src/extension/serviceWorker.ts b/ts/packages/agents/browser/src/extension/serviceWorker.ts
@@ -589,6 +589,7 @@ async function getTabHTMLFragments(
     targetTab: chrome.tabs.Tab,
     fullSize: boolean,
     downloadAsFile: boolean,
+    extractText: boolean,
     maxFragmentSize: 16000,
 ) {
     const frames = await chrome.webNavigation.getAllFrames({
@@ -612,15 +613,18 @@ async function getTabHTMLFragments(
                 );
 
                 if (frameHTML) {
-                    const frameText = await chrome.tabs.sendMessage(
-                        targetTab.id!,
-                        {
-                            type: "get_page_text",
-                            inputHtml: frameHTML,
-                            frameId: frames[i].frameId,
-                        },
-                        { frameId: frames[i].frameId },
-                    );
+                    let frameText = "";
+                    if (extractText) {
+                        frameText = await chrome.tabs.sendMessage(
+                            targetTab.id!,
+                            {
+                                type: "get_page_text",
+                                inputHtml: frameHTML,
+                                frameId: frames[i].frameId,
+                            },
+                            { frameId: frames[i].frameId },
+                        );
+                    }
 
                     if (downloadAsFile) {
                         await downloadStringAsFile(
@@ -1126,6 +1130,7 @@ async function runBrowserAction(action: any) {
                 targetTab,
                 action.parameters?.fullHTML,
                 action.parameters?.downloadAsFile,
+                action.parameters?.extractText,
                 16000,
             );
             break;

diff --git a/ts/packages/shell/src/preload/webView.ts b/ts/packages/shell/src/preload/webView.ts
@@ -83,7 +83,10 @@ export async function awaitPageLoad() {
     */
 }
 
-export async function getTabHTMLFragments(fullSize: boolean) {
+export async function getTabHTMLFragments(
+    fullSize: boolean,
+    extactText: boolean,
+) {
     let htmlFragments: any[] = [];
     let htmlPromises: Promise<any>[] = [];
 
@@ -131,15 +134,18 @@ export async function getTabHTMLFragments(fullSize: boolean) {
     for (let i = 0; i < htmlResults.length; i++) {
         const frameHTML = htmlResults[i];
         if (frameHTML) {
-            const frameText = await sendScriptAction(
-                {
-                    type: "get_page_text",
-                    inputHtml: frameHTML,
-                    frameId: i,
-                },
-                1000,
-                frames[i],
-            );
+            let frameText = "";
+            if (extactText) {
+                frameText = await sendScriptAction(
+                    {
+                        type: "get_page_text",
+                        inputHtml: frameHTML,
+                        frameId: i,
+                    },
+                    1000,
+                    frames[i],
+                );
+            }
 
             htmlFragments.push({
                 frameId: i,
@@ -303,6 +309,7 @@ async function runBrowserAction(action: any) {
         case "getHTML": {
             responseObject = await getTabHTMLFragments(
                 action.parameters.fullHTML,
+                action.parameters?.extractText,
             );
 
             break;