From fdaafa8afcb876b95bf9ae62311d4c7fb3c851f2 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Tue, 10 Jun 2025 06:41:39 -0700 Subject: [PATCH] click by x,y coordinate fallback when main method fails --- browser_use/browser/session.py | 14 ++++++++++++++ browser_use/controller/service.py | 7 ++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/browser_use/browser/session.py b/browser_use/browser/session.py index 31a16abec..2aaf9d91c 100644 --- a/browser_use/browser/session.py +++ b/browser_use/browser/session.py @@ -1490,6 +1490,20 @@ class BrowserSession(BaseModel): except URLNotAllowedError as e: raise e except Exception as e: + # Final fallback - try clicking by coordinates if available + if element_node.viewport_coordinates and element_node.viewport_coordinates.center: + try: + self.logger.warning( + f'⚠️ Element click failed, falling back to coordinate click at ({element_node.viewport_coordinates.center.x}, {element_node.viewport_coordinates.center.y})' + ) + await page.mouse.click( + element_node.viewport_coordinates.center.x, element_node.viewport_coordinates.center.y + ) + await page.wait_for_load_state() + await self._check_and_handle_navigation(page) + return None # Success + except Exception as coord_e: + self.logger.error(f'Coordinate click also failed: {type(coord_e).__name__}: {coord_e}') raise Exception(f'Failed to click element: {type(e).__name__}: {e}') except URLNotAllowedError as e: diff --git a/browser_use/controller/service.py b/browser_use/controller/service.py index 201820c26..46f818d2d 100644 --- a/browser_use/controller/service.py +++ b/browser_use/controller/service.py @@ -139,7 +139,12 @@ class Controller(Generic[Context]): selector_map = await browser_session.get_selector_map() if params.index not in selector_map: - raise Exception(f'Element with index {params.index} does not exist - retry or use alternative actions') + # Return informative message with the new state instead of error + max_index = max(selector_map.keys()) if selector_map else -1 + return ActionResult( + extracted_content=f'Element with index {params.index} does not exist. Page has {len(selector_map)} interactive elements (indices 0-{max_index}). State has been refreshed - please use the updated element indices.', + include_in_memory=True, + ) element_node = await browser_session.get_dom_element_by_index(params.index) initial_pages = len(browser_session.tabs)