From 21dec0dc7fd3f02b64f2a48cd2f3f8b2cb99775f Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 24 Mar 2025 10:59:48 -0700 Subject: [PATCH] skip iframes that already have tabs open --- browser_use/browser/context.py | 6 ++++-- browser_use/controller/service.py | 9 ++++----- browser_use/dom/buildDomTree.js | 28 ++++++++++++++-------------- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/browser_use/browser/context.py b/browser_use/browser/context.py index eb3fb046f..66ade677b 100644 --- a/browser_use/browser/context.py +++ b/browser_use/browser/context.py @@ -762,7 +762,7 @@ class BrowserContext: # Check if current page is still valid, if not switch to another available page try: page = await self.get_current_page() - # Test if page and its iframes are still accessible + # Test if page is still accessible await page.evaluate('1') except Exception as e: logger.debug(f'Current page is no longer accessible: {str(e)}') @@ -790,6 +790,8 @@ class BrowserContext: # mark the titles of the new tabs so the LLM knows to check them for additional content iframe_urls = await dom_service.get_cross_origin_iframes() for url in iframe_urls: + if url in [tab.url for tab in tabs_info]: + continue # skip if the iframe if we already have it open in a tab await self.create_new_tab(url) tabs_info.append( TabInfo( @@ -1201,7 +1203,7 @@ class BrowserContext: tab_info = TabInfo(page_id=page_id, url=page.url, title=await asyncio.wait_for(page.title(), timeout=1)) tabs_info.append(tab_info) except asyncio.TimeoutError: - # page.title with hang forever on tabs that are frozen or about:blank + # page.title() can hang forever on tabs that are crashed/dissapeared/about:blank # we dont want to try automating those tabs because they will hang the whole script logger.debug('Failed to get tab info for tab: %s (ignoring)', page_id) continue diff --git a/browser_use/controller/service.py b/browser_use/controller/service.py index 96f9b62a0..bcda27ce3 100644 --- a/browser_use/controller/service.py +++ b/browser_use/controller/service.py @@ -1,9 +1,8 @@ import asyncio -import re -import json import enum import json import logging +import re from typing import Dict, Generic, Optional, Type, TypeVar from langchain_core.language_models.chat_models import BaseChatModel @@ -166,7 +165,7 @@ class Controller(Generic[Context]): logger.info(msg) logger.debug(f'Element xpath: {element_node.xpath}') return ActionResult(extracted_content=msg, include_in_memory=True) - + # Save PDF @self.registry.action( 'Save the current page as a PDF file', @@ -179,7 +178,7 @@ class Controller(Generic[Context]): await page.emulate_media('screen') await page.pdf(path=sanitized_filename, format='A4', print_background=False) - msg = f"Saving page with URL {page.url} as PDF to ./{sanitized_filename}" + msg = f'Saving page with URL {page.url} as PDF to ./{sanitized_filename}' logger.info(msg) return ActionResult(extracted_content=msg, include_in_memory=True) @@ -211,7 +210,7 @@ class Controller(Generic[Context]): content = markdownify.markdownify(await page.content()) - # append iframe content into the page so it's accessible to the LLM too + # manually append iframe text into the content so it's readable by the LLM (includes cross-origin iframes) for iframe in page.frames: if iframe.url != page.url and not iframe.url.startswith('data:'): content += f'\n\nIFRAME {iframe.url}:\n' diff --git a/browser_use/dom/buildDomTree.js b/browser_use/dom/buildDomTree.js index fb790f426..9286c097d 100644 --- a/browser_use/dom/buildDomTree.js +++ b/browser_use/dom/buildDomTree.js @@ -1,5 +1,12 @@ -({ doHighlightElements=true, focusHighlightIndex=-1, viewportExpansion=0, debugMode=false, indexOffset=0 }={}) => { - // console.log('indexOffset', indexOffset) +( + args = { + doHighlightElements: true, + focusHighlightIndex: -1, + viewportExpansion: 0, + debugMode: false, + } +) => { + const { doHighlightElements, focusHighlightIndex, viewportExpansion, debugMode } = args; let highlightIndex = 0; // Reset highlight index // Add timing stack to handle recursion @@ -176,7 +183,7 @@ */ const DOM_HASH_MAP = {}; - const ID = { current: indexOffset }; + const ID = { current: 0 }; const HIGHLIGHT_CONTAINER_ID = "playwright-highlight-container"; @@ -768,8 +775,8 @@ element.hasAttribute("role") || element.hasAttribute("tabindex") || element.hasAttribute("aria-") || - element.hasAttribute("data-action") || - element.getAttribute("contenteditable") == "true"; + element.hasAttribute("data-action"); + return hasQuickInteractiveAttr; } @@ -793,10 +800,6 @@ return null; } - // FOR DEBUGGING: mark the node as having been processed by buildDomTree.js - // so that playwright can tell if it's been processed or not - // node.setAttribute('data-browser-use-built-dom-tree', 'true'); - // Special handling for root node (body) if (node === document.body) { const nodeData = { @@ -933,9 +936,6 @@ const domElement = buildDomTree(child, node); if (domElement) nodeData.children.push(domElement); } - // mark the iframe as having been processed by buildDomTree.js - // so that playwright can tell if got processed already or if it needs special handling - node.contentWindow._loadedBrowserUseBuildDomTree = true; } } catch (e) { console.warn("Unable to access iframe:", e); @@ -1050,6 +1050,6 @@ } return debugMode ? - { rootId, map: DOM_HASH_MAP, perfMetrics: PERF_METRICS, indexOffset } : - { rootId, map: DOM_HASH_MAP, indexOffset }; + { rootId, map: DOM_HASH_MAP, perfMetrics: PERF_METRICS } : + { rootId, map: DOM_HASH_MAP }; };