eliott/browser-use
1
0
Fork 0
mirror of https://github.com/browser-use/browser-use synced 2026-05-06 17:52:15 +02:00
Code Issues Packages Projects Releases Wiki Activity

Remove outdated methods

Browse Source
This commit is contained in:
Magnus Müller
2025-08-31 11:31:32 -07:00
parent 33f69b1525
commit a652335316
4 changed files with 1 additions and 497 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
browser_use/browser/watchdogs/dom_watchdog.py
View File

@@ -347,85 +347,6 @@ class DOMWatchdog(BaseWatchdog):
recent_events=None,
)
async def _build_dom_tree(self, previous_state: SerializedDOMState | None = None) -> SerializedDOMState:
"""Internal method to build and serialize DOM tree.
This is the actual implementation that does the work, called by both
on_BrowserStateRequestEvent.
Returns:
SerializedDOMState with serialized DOM and selector map
"""
try:
self.logger.debug('🔍 DOMWatchdog._build_dom_tree: STARTING DOM tree build')
# Remove any existing highlights before building new DOM
try:
self.logger.debug('🔍 DOMWatchdog._build_dom_tree: Removing existing highlights...')
await self.browser_session.remove_highlights()
# self.logger.debug('🔍 DOMWatchdog._build_dom_tree: ✅ Highlights removed')
except Exception as e:
self.logger.debug(f'🔍 DOMWatchdog._build_dom_tree: Failed to remove existing highlights: {e}')
# Create or reuse DOM service
if self._dom_service is None:
# self.logger.debug('🔍 DOMWatchdog._build_dom_tree: Creating DomService...')
self._dom_service = DomService(
browser_session=self.browser_session,
logger=self.logger,
cross_origin_iframes=self.browser_session.browser_profile.cross_origin_iframes,
)
# self.logger.debug('🔍 DOMWatchdog._build_dom_tree: ✅ DomService created')
# else:
# self.logger.debug('🔍 DOMWatchdog._build_dom_tree: Reusing existing DomService')
# Get serialized DOM tree using the service
self.logger.debug('🔍 DOMWatchdog._build_dom_tree: Calling DomService.get_serialized_dom_tree...')
start = time.time()
self.current_dom_state, self.enhanced_dom_tree, timing_info = await self._dom_service.get_serialized_dom_tree(
previous_cached_state=previous_state,
)
end = time.time()
self.logger.debug('🔍 DOMWatchdog._build_dom_tree: ✅ DomService.get_serialized_dom_tree completed')
self.logger.debug(f'Time taken to get DOM tree: {end - start} seconds')
self.logger.debug(f'Timing breakdown: {timing_info}')
# Update selector map for other watchdogs
self.logger.debug('🔍 DOMWatchdog._build_dom_tree: Updating selector maps...')
self.selector_map = self.current_dom_state.selector_map
# Update BrowserSession's cached selector map
if self.browser_session:
self.browser_session.update_cached_selector_map(self.selector_map)
self.logger.debug(f'🔍 DOMWatchdog._build_dom_tree: ✅ Selector maps updated, {len(self.selector_map)} elements')
# Inject highlighting for visual feedback if we have elements
if self.selector_map and self._dom_service and self.browser_session.browser_profile.highlight_elements:
try:
self.logger.debug('🔍 DOMWatchdog._build_dom_tree: Injecting highlighting script...')
from browser_use.dom.debug.highlights import inject_highlighting_script
await inject_highlighting_script(self._dom_service, self.selector_map)
self.logger.debug(
f'🔍 DOMWatchdog._build_dom_tree: ✅ Injected highlighting for {len(self.selector_map)} elements'
)
except Exception as e:
self.logger.debug(f'🔍 DOMWatchdog._build_dom_tree: Failed to inject highlighting: {e}')
elif self.selector_map and self._dom_service and not self.browser_session.browser_profile.highlight_elements:
self.logger.debug('🔍 DOMWatchdog._build_dom_tree: Skipping highlighting injection - highlight_elements=False')
self.logger.debug('🔍 DOMWatchdog._build_dom_tree: ✅ COMPLETED DOM tree build')
return self.current_dom_state
except Exception as e:
self.logger.error(f'Failed to build DOM tree: {e}')
self.event_bus.dispatch(
BrowserErrorEvent(
error_type='DOMBuildFailed',
message=str(e),
)
)
raise
@time_execution_async('build_dom_tree_without_highlights')
@observe_debug(ignore_input=True, ignore_output=True, name='build_dom_tree_without_highlights')
async def _build_dom_tree_without_highlights(self, previous_state: SerializedDOMState | None = None) -> SerializedDOMState:
@@ -622,7 +543,7 @@ class DOMWatchdog(BaseWatchdog):
"""
if not self.selector_map:
# Build DOM if not cached
await self._build_dom_tree()
await self._build_dom_tree_without_highlights()
return self.selector_map.get(index) if self.selector_map else None

405
browser_use/dom/debug/highlights.py
View File

@@ -1,405 +0,0 @@
# 100% vibe coded
import json
import logging
import traceback
from browser_use.dom.service import DomService
from browser_use.dom.views import DOMSelectorMap
logger = logging.getLogger(__name__)
def convert_dom_selector_map_to_highlight_format(selector_map: DOMSelectorMap) -> list[dict]:
"""Convert DOMSelectorMap to the format expected by the highlighting script."""
elements = []
for interactive_index, node in selector_map.items():
# Get bounding box using absolute position (includes iframe translations) if available
bbox = None
if node.absolute_position:
# Use absolute position which includes iframe coordinate translations
rect = node.absolute_position
bbox = {'x': rect.x, 'y': rect.y, 'width': rect.width, 'height': rect.height}
# Only include elements with valid bounding boxes (following working implementation)
if bbox and bbox.get('width', 0) > 0 and bbox.get('height', 0) > 0:
element = {
'x': bbox['x'],
'y': bbox['y'],
'width': bbox['width'],
'height': bbox['height'],
'interactive_index': interactive_index,
'element_name': node.node_name,
'is_clickable': node.snapshot_node.is_clickable if node.snapshot_node else True,
'is_scrollable': getattr(node, 'is_scrollable', False),
'attributes': node.attributes or {},
'frame_id': getattr(node, 'frame_id', None),
'node_id': node.node_id,
'backend_node_id': node.backend_node_id,
'xpath': node.xpath,
'text_content': node.get_all_children_text()[:50]
if hasattr(node, 'get_all_children_text')
else node.node_value[:50],
}
elements.append(element)
else:
# Skip elements without valid bounding boxes for now
# Could add fallback positioning here if needed
pass
return elements
async def remove_highlighting_script(dom_service: DomService) -> None:
"""Remove all browser-use highlighting elements from the page."""
try:
# Get CDP client and session ID
cdp_session = await dom_service.browser_session.get_or_create_cdp_session()
logger.debug('🧹 Removing browser-use highlighting elements')
# Create script to remove all highlights
script = """
(function() {
// Remove any existing highlights - be thorough
const existingHighlights = document.querySelectorAll('[data-browser-use-highlight]');
console.log('Removing', existingHighlights.length, 'browser-use highlight elements');
existingHighlights.forEach(el => el.remove());
// Also remove by ID in case selector missed anything
const highlightContainer = document.getElementById('browser-use-debug-highlights');
if (highlightContainer) {
console.log('Removing highlight container by ID');
highlightContainer.remove();
}
// Final cleanup - remove any orphaned tooltips
const orphanedTooltips = document.querySelectorAll('[data-browser-use-highlight="tooltip"]');
orphanedTooltips.forEach(el => el.remove());
})();
"""
# Execute the removal script via CDP
await cdp_session.cdp_client.send.Runtime.evaluate(
params={'expression': script, 'returnByValue': True}, session_id=cdp_session.session_id
)
except Exception as e:
logger.exception(f'Error removing highlighting elements: {e}', exc_info=True)
async def inject_highlighting_script(dom_service: DomService, interactive_elements: DOMSelectorMap) -> None:
"""Inject JavaScript to highlight interactive elements with detailed hover tooltips that work around CSP restrictions."""
if not interactive_elements:
logger.debug('⚠️ No interactive elements to highlight')
return
try:
# Convert DOMSelectorMap to the format expected by the JavaScript
converted_elements = convert_dom_selector_map_to_highlight_format(interactive_elements)
logger.debug(f'Creating CSP-safe highlighting for {len(converted_elements)} elements')
# ALWAYS remove any existing highlights first to prevent double-highlighting
await remove_highlighting_script(dom_service)
# Add a small delay to ensure removal completes
import asyncio
await asyncio.sleep(0.05)
# Create CSP-safe highlighting script using DOM methods instead of innerHTML
# Uses outline-only highlights with reasonable z-index to avoid blocking page content
script = f"""
(function() {{
// Interactive elements data with reasoning
const interactiveElements = {json.dumps(converted_elements)};
console.log('=== BROWSER-USE HIGHLIGHTING ===');
console.log('Highlighting', interactiveElements.length, 'interactive elements');
// Double-check: Remove any existing highlight container first to prevent duplicates
const existingContainer = document.getElementById('browser-use-debug-highlights');
if (existingContainer) {{
console.log('⚠️ Found existing highlight container, removing it first');
existingContainer.remove();
}}
// Also remove any stray highlight elements
const strayHighlights = document.querySelectorAll('[data-browser-use-highlight]');
if (strayHighlights.length > 0) {{
console.log('⚠️ Found', strayHighlights.length, 'stray highlight elements, removing them');
strayHighlights.forEach(el => el.remove());
}}
// Use a high but reasonable z-index to be visible without covering important content
// High enough for most content but not maximum to avoid blocking critical popups/modals
const HIGHLIGHT_Z_INDEX = 2147483647; // Maximum z-index for CSS (2^31-1)
// Create container for all highlights - use fixed positioning without scroll calculations
const container = document.createElement('div');
container.id = 'browser-use-debug-highlights';
container.setAttribute('data-browser-use-highlight', 'container');
container.style.cssText = `
position: fixed;
top: 0;
left: 0;
width: 100vw;
height: 100vh;
pointer-events: none;
z-index: ${{HIGHLIGHT_Z_INDEX}};
overflow: visible;
margin: 0;
padding: 0;
border: none;
outline: none;
box-shadow: none;
background: none;
font-family: inherit;
`;
// Helper function to create text nodes safely (CSP-friendly)
function createTextElement(tag, text, styles) {{
const element = document.createElement(tag);
element.textContent = text;
if (styles) element.style.cssText = styles;
return element;
}}
// Add enhanced highlights with detailed tooltips
interactiveElements.forEach((element, index) => {{
const highlight = document.createElement('div');
highlight.setAttribute('data-browser-use-highlight', 'element');
highlight.setAttribute('data-element-id', element.interactive_index);
highlight.style.cssText = `
position: absolute;
left: ${{element.x}}px;
top: ${{element.y}}px;
width: ${{element.width}}px;
height: ${{element.height}}px;
outline: 2px solid #4a90e2;
outline-offset: -2px;
background: transparent;
pointer-events: none;
box-sizing: content-box;
transition: outline 0.2s ease;
margin: 0;
padding: 0;
border: none;
`;
// Enhanced label with interactive index
const label = createTextElement('div', element.interactive_index, `
position: absolute;
top: -20px;
left: 0;
background-color: #4a90e2;
color: white;
padding: 2px 6px;
font-size: 11px;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;
font-weight: bold;
border-radius: 3px;
white-space: nowrap;
z-index: ${{HIGHLIGHT_Z_INDEX + 1}};
box-shadow: 0 2px 4px rgba(0,0,0,0.3);
border: none;
outline: none;
margin: 0;
line-height: 1.2;
`);
// Enhanced tooltip with detailed reasoning (CSP-safe)
const tooltip = document.createElement('div');
tooltip.setAttribute('data-browser-use-highlight', 'tooltip');
tooltip.style.cssText = `
position: absolute;
top: -160px;
left: 50%;
transform: translateX(-50%);
background-color: rgba(0, 0, 0, 0.95);
color: white;
padding: 12px 16px;
font-size: 12px;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;
border-radius: 8px;
white-space: nowrap;
z-index: ${{HIGHLIGHT_Z_INDEX + 2}};
opacity: 0;
visibility: hidden;
transition: all 0.3s ease;
box-shadow: 0 6px 20px rgba(0,0,0,0.5);
border: 1px solid #666;
max-width: 400px;
white-space: normal;
line-height: 1.4;
min-width: 200px;
margin: 0;
`;
// Build detailed tooltip content with reasoning (CSP-safe DOM creation)
const reasoning = element.reasoning || {{}};
const confidence = reasoning.confidence || 'UNKNOWN';
const primaryReason = reasoning.primary_reason || 'unknown';
const reasons = reasoning.reasons || [];
const elementType = reasoning.element_type || element.element_name || 'UNKNOWN';
// Determine confidence color and styling
let confidenceColor = '#4a90e2';
let confidenceIcon = '🔍';
let outlineColor = '#4a90e2';
let shadowColor = '#4a90e2';
if (confidence === 'HIGH') {{
confidenceColor = '#28a745';
confidenceIcon = '✅';
outlineColor = '#28a745';
shadowColor = '#28a745';
}} else if (confidence === 'MEDIUM') {{
confidenceColor = '#ffc107';
confidenceIcon = '⚠️';
outlineColor = '#ffc107';
shadowColor = '#ffc107';
}} else {{
confidenceColor = '#fd7e14';
confidenceIcon = '❓';
outlineColor = '#fd7e14';
shadowColor = '#fd7e14';
}}
// Create tooltip header
const header = createTextElement('div', `${{confidenceIcon}} [${{element.interactive_index}}] ${{elementType.toUpperCase()}}`, `
color: ${{confidenceColor}};
font-weight: bold;
font-size: 13px;
margin-bottom: 8px;
border-bottom: 1px solid #666;
padding-bottom: 4px;
`);
// Create confidence indicator
const confidenceDiv = createTextElement('div', `${{confidence}} CONFIDENCE`, `
color: ${{confidenceColor}};
font-size: 11px;
font-weight: bold;
margin-bottom: 8px;
`);
// Create primary reason
const primaryReasonDiv = createTextElement('div', `Primary: ${{primaryReason.replace('_', ' ').toUpperCase()}}`, `
color: #fff;
font-size: 11px;
margin-bottom: 6px;
font-weight: bold;
`);
// Create reasons list
const reasonsContainer = document.createElement('div');
reasonsContainer.style.cssText = `
font-size: 10px;
color: #ccc;
margin-top: 4px;
`;
if (reasons.length > 0) {{
const reasonsTitle = createTextElement('div', 'Evidence:', `
color: #fff;
font-size: 10px;
margin-bottom: 4px;
font-weight: bold;
`);
reasonsContainer.appendChild(reasonsTitle);
reasons.slice(0, 4).forEach(reason => {{
const reasonDiv = createTextElement('div', `• ${{reason}}`, `
color: #ccc;
font-size: 10px;
margin-bottom: 2px;
padding-left: 4px;
`);
reasonsContainer.appendChild(reasonDiv);
}});
if (reasons.length > 4) {{
const moreDiv = createTextElement('div', `... and ${{reasons.length - 4}} more`, `
color: #999;
font-size: 9px;
font-style: italic;
margin-top: 2px;
`);
reasonsContainer.appendChild(moreDiv);
}}
}} else {{
const noReasonsDiv = createTextElement('div', 'No specific evidence found', `
color: #999;
font-size: 10px;
font-style: italic;
`);
reasonsContainer.appendChild(noReasonsDiv);
}}
// Add bounding box info
const boundsDiv = createTextElement('div', `Position: (${{Math.round(element.x)}}, ${{Math.round(element.y)}}) Size: ${{Math.round(element.width)}}×${{Math.round(element.height)}}`, `
color: #888;
font-size: 9px;
margin-top: 8px;
border-top: 1px solid #444;
padding-top: 4px;
`);
// Assemble tooltip
tooltip.appendChild(header);
tooltip.appendChild(confidenceDiv);
tooltip.appendChild(primaryReasonDiv);
tooltip.appendChild(reasonsContainer);
tooltip.appendChild(boundsDiv);
// Set highlight colors based on confidence (outline only)
highlight.style.outline = `2px solid ${{outlineColor}}`;
label.style.backgroundColor = outlineColor;
// Add subtle hover effects (outline only, no background)
highlight.addEventListener('mouseenter', () => {{
highlight.style.outline = '3px solid #ff6b6b';
highlight.style.outlineOffset = '-1px';
tooltip.style.opacity = '1';
tooltip.style.visibility = 'visible';
label.style.backgroundColor = '#ff6b6b';
label.style.transform = 'scale(1.1)';
}});
highlight.addEventListener('mouseleave', () => {{
highlight.style.outline = `2px solid ${{outlineColor}}`;
highlight.style.outlineOffset = '-2px';
tooltip.style.opacity = '0';
tooltip.style.visibility = 'hidden';
label.style.backgroundColor = outlineColor;
label.style.transform = 'scale(1)';
}});
highlight.appendChild(tooltip);
highlight.appendChild(label);
container.appendChild(highlight);
}});
// Add container to document
document.body.appendChild(container);
console.log('Highlighting complete');
}})();
"""
cdp_session = await dom_service.browser_session.get_or_create_cdp_session()
# Inject the enhanced CSP-safe script via CDP
await cdp_session.cdp_client.send.Runtime.evaluate(
params={'expression': script, 'returnByValue': True}, session_id=cdp_session.session_id
)
logger.debug(f'Enhanced CSP-safe highlighting injected for {len(converted_elements)} elements')
except Exception as e:
logger.debug(f'❌ Error injecting enhanced highlighting script: {e}')
traceback.print_exc()

7
browser_use/dom/playground/extraction.py
View File

@@ -11,7 +11,6 @@ from browser_use.agent.prompts import AgentMessagePrompt
from browser_use.browser import BrowserProfile, BrowserSession
from browser_use.browser.events import ClickElementEvent, TypeTextEvent
from browser_use.browser.profile import ViewportSize
from browser_use.dom.debug.highlights import inject_highlighting_script, remove_highlighting_script
from browser_use.dom.service import DomService
from browser_use.dom.views import DEFAULT_INCLUDE_ATTRIBUTES
from browser_use.filesystem.file_system import FileSystem
@@ -115,12 +114,8 @@ async def test_focus_vs_all_elements():
last_clicked_index = None # Track the index for text input
while True:
try:
await remove_highlighting_script(dom_service)
# all_elements_state = await dom_service.get_serialized_dom_tree()
# await inject_highlighting_script(dom_service, all_elements_state.selector_map)
website_type = 'DIFFICULT' if website in difficult_websites else 'SAMPLE'
print(f'\n{"=" * 60}')
print(f'[{current_website_index + 1}/{len(websites)}] [{website_type}] Testing: {website}')
@@ -144,8 +139,6 @@ async def test_focus_vs_all_elements():
# Combine all timing info
all_timing = {'get_state_summary_total': get_state_time, **timing_info}
await inject_highlighting_script(dom_service, all_elements_state.dom_state.selector_map)
selector_map = all_elements_state.dom_state.selector_map
total_elements = len(selector_map.keys())
print(f'Total number of elements: {total_elements}')

5
browser_use/dom/playground/tree.py
View File

@@ -6,7 +6,6 @@ import aiofiles
from browser_use.browser import BrowserProfile, BrowserSession
from browser_use.browser.types import ViewportSize
from browser_use.dom.debug.highlights import inject_highlighting_script, remove_highlighting_script
from browser_use.dom.service import DomService
from browser_use.dom.views import EnhancedDOMTreeNode
@@ -46,8 +45,6 @@ async def main():
while True:
async with DomService(browser) as dom_service:
await remove_highlighting_script(dom_service)
start = time.time()
# Get current target ID from browser session
if browser.agent_focus and browser.agent_focus.target_id:
@@ -122,8 +119,6 @@ async def main():
print('saved snapshot to tmp/snapshot.json')
print('saved ax tree to tmp/ax_tree.json')
await inject_highlighting_script(dom_service, serialized_dom_state.selector_map)
input('Done. Press Enter to continue...')