"""
Learning Agent that extends browser-use with experience-based learning.
This agent wraps the browser-use Agent to capture successful workflows
and replay them efficiently without LLM calls.
"""
import asyncio
import logging
from typing import Any, Dict, List, Optional
import time
from pathlib import Path
import sys
import os
# Add parent directory to path to import browser-use
sys.path.append(str(Path(__file__).parent.parent))
from browser_use import Agent, Browser
from browser_use.agent.views import AgentOutput, ActionResult
from browser_use.browser.views import BrowserStateSummary
from .workflow import Workflow, WorkflowStep, ActionType
from .knowledge_base import KnowledgeBase
from .replay import WorkflowReplayer
logger = logging.getLogger(__name__)
class LearningAgent:
"""
An agent that learns from experience and can replay learned workflows.
This agent:
1. Attempts to match tasks to learned workflows
2. Falls back to browser-use Agent for new tasks
3. Captures successful executions as new workflows
4. Improves over time by building a knowledge base
"""
def __init__(self,
task: str,
llm: Any,
browser: Optional[Browser] = None,
knowledge_base_path: str = "./knowledge_base",
headless: bool = False,
**agent_kwargs):
"""
Initialize the learning agent.
Args:
task: The task to be performed
llm: Language model to use (browser-use compatible)
browser: Browser instance (optional)
knowledge_base_path: Path to store learned workflows
headless: Whether to run browser in headless mode for replay
**agent_kwargs: Additional arguments for browser-use Agent
"""
self.task = task
self.llm = llm
self.browser = browser
self.headless = headless
# Initialize knowledge base
self.knowledge_base = KnowledgeBase(knowledge_base_path)
# Initialize workflow replayer
self.replayer = WorkflowReplayer(headless=headless)
# Workflow capture state
self.current_workflow: Optional[Workflow] = None
self.is_learning = False
self.captured_steps: List[Dict[str, Any]] = []
# Browser-use agent (lazy initialization)
self._agent: Optional[Agent] = None
self._agent_kwargs = agent_kwargs
# Metrics
self.metrics = {
"llm_calls": 0,
"replay_used": False,
"execution_time": 0,
"success": False
}
@property
def agent(self) -> Agent:
"""Lazy initialization of browser-use agent."""
if self._agent is None:
# Create agent with step callback for capturing
self._agent = Agent(
task=self.task,
llm=self.llm,
browser=self.browser,
**self._agent_kwargs
)
# Store original step method
self._original_step = self._agent.step
# Monkey-patch the step method to capture actions
self._agent.step = self._wrapped_step
return self._agent
async def _wrapped_step(self, step_info=None):
"""Wrapped step method that captures workflow information."""
# Call original step
await self._original_step(step_info)
# Capture step information if learning
if self.is_learning:
await self._capture_step()
async def _capture_step(self):
"""Capture the current step for workflow learning."""
try:
# Get the last action and result from agent state
if self.agent.state.last_model_output and self.agent.state.last_result:
model_output = self.agent.state.last_model_output
results = self.agent.state.last_result
# Get browser state for element information
browser_state = await self.agent.browser_session.get_browser_state_summary()
# Process each action in the step
for i, (action, result) in enumerate(zip(model_output.action, results)):
if result and not result.error:
# Extract action details
action_data = self._extract_action_data(action, result, browser_state)
if action_data:
self.captured_steps.append(action_data)
logger.debug(f"Captured step: {action_data['type']}")
except Exception as e:
logger.error(f"Failed to capture step: {e}")
def _extract_action_data(self, action: Any, result: ActionResult, browser_state: BrowserStateSummary) -> Optional[Dict[str, Any]]:
"""
Extract action data for workflow capture.
Args:
action: The action object from browser-use
result: The result of the action
browser_state: Current browser state
Returns:
Dictionary containing action data, or None if extraction fails
"""
try:
action_dict = action.model_dump() if hasattr(action, 'model_dump') else {}
# Determine action type
action_type = None
parameters = {}
element_info = None
# Parse different action types
if 'go_to_url' in action_dict:
action_type = ActionType.NAVIGATE
parameters = {'url': action_dict['go_to_url'].get('url')}
elif 'click_element' in action_dict:
action_type = ActionType.CLICK
click_data = action_dict['click_element']
parameters = {
'while_holding_ctrl': click_data.get('while_holding_ctrl', False)
}
# Get element info from selector map
index = click_data.get('index')
if index and browser_state.dom_state.selector_map:
element_info = self._get_element_info(index, browser_state.dom_state.selector_map)
elif 'input_text' in action_dict:
action_type = ActionType.INPUT_TEXT
input_data = action_dict['input_text']
parameters = {
'text': input_data.get('text', ''),
'clear_existing': input_data.get('clear_existing', True)
}
# Get element info
index = input_data.get('index')
if index and browser_state.dom_state.selector_map:
element_info = self._get_element_info(index, browser_state.dom_state.selector_map)
elif 'select_dropdown_option' in action_dict:
action_type = ActionType.SELECT_OPTION
select_data = action_dict['select_dropdown_option']
parameters = {
'text': select_data.get('text', '')
}
index = select_data.get('index')
if index and browser_state.dom_state.selector_map:
element_info = self._get_element_info(index, browser_state.dom_state.selector_map)
elif 'scroll' in action_dict:
action_type = ActionType.SCROLL
scroll_data = action_dict['scroll']
parameters = {
'down': scroll_data.get('down', True),
'num_pages': scroll_data.get('num_pages', 1)
}
elif 'upload_file' in action_dict:
action_type = ActionType.UPLOAD_FILE
upload_data = action_dict['upload_file']
parameters = {
'path': upload_data.get('path', '')
}
index = upload_data.get('index')
if index and browser_state.dom_state.selector_map:
element_info = self._get_element_info(index, browser_state.dom_state.selector_map)
elif 'done' in action_dict:
# Skip done action for workflow
return None
if action_type:
return {
'type': action_type,
'parameters': parameters,
'element_info': element_info,
'url': browser_state.url
}
except Exception as e:
logger.error(f"Failed to extract action data: {e}")
return None
def _get_element_info(self, index: int, selector_map: Dict) -> Optional[Dict[str, Any]]:
"""
Get element information from selector map.
Args:
index: Element index
selector_map: Browser-use selector map
Returns:
Dictionary containing element selectors and attributes
"""
try:
if index in selector_map:
element = selector_map[index]
# Extract stable selectors
info = {
'xpath': getattr(element, 'xpath', None),
'attributes': {}
}
# Get important attributes
if hasattr(element, 'attributes') and element.attributes:
for attr in ['id', 'name', 'class', 'type', 'role', 'aria-label', 'data-testid']:
if attr in element.attributes:
info['attributes'][attr] = element.attributes[attr]
return info
except Exception as e:
logger.error(f"Failed to get element info: {e}")
return None
async def run(self, max_steps: int = 100) -> Dict[str, Any]:
"""
Run the learning agent to complete the task.
Args:
max_steps: Maximum steps for browser-use agent
Returns:
Dictionary containing execution results and metrics
"""
start_time = time.time()
try:
# Check if we have a learned workflow for this task
match = self.knowledge_base.find_workflow_for_task(self.task)
if match and match.confidence > 0.6:
# Use learned workflow
logger.info(f"Found matching workflow: '{match.workflow.intent}' "
f"(confidence: {match.confidence:.2f})")
logger.info(f"Match reason: {match.match_reason}")
result = await self._run_with_replay(match.workflow)
# Update metrics
self.knowledge_base.update_workflow_metrics(
match.workflow.workflow_id,
success=result['success'],
execution_time=result['execution_time'],
model_calls_saved=result['model_calls_saved']
)
self.metrics['replay_used'] = True
self.metrics['success'] = result['success']
# If replay failed, fall back to learning mode
if not result['success']:
logger.warning("Replay failed, falling back to learning mode")
result = await self._run_with_learning(max_steps)
else:
# No matching workflow, run in learning mode
logger.info("No matching workflow found, running in learning mode")
result = await self._run_with_learning(max_steps)
finally:
self.metrics['execution_time'] = time.time() - start_time
# Log performance comparison
if self.metrics['replay_used']:
logger.info(f"Task completed with replay in {self.metrics['execution_time']:.2f}s")
logger.info(f"Model calls saved: {result.get('model_calls_saved', 0)}")
else:
logger.info(f"Task completed with learning in {self.metrics['execution_time']:.2f}s")
logger.info(f"LLM calls made: {self.metrics['llm_calls']}")
return self.metrics
async def _run_with_replay(self, workflow: Workflow) -> Dict[str, Any]:
"""
Run task using a learned workflow.
Args:
workflow: The workflow to replay
Returns:
Execution results
"""
logger.info("Replaying learned workflow...")
# Extract parameters from task if needed
parameters = self._extract_task_parameters(self.task, workflow)
# Setup replayer
await self.replayer.setup()
try:
# Replay workflow
result = await self.replayer.replay_workflow(
workflow,
parameters=parameters
)
logger.info(f"Replay completed: {result['steps_completed']}/{result['total_steps']} steps")
return result
finally:
await self.replayer.cleanup()
async def _run_with_learning(self, max_steps: int) -> Dict[str, Any]:
"""
Run task with browser-use agent and capture workflow.
Args:
max_steps: Maximum steps for agent
Returns:
Execution results
"""
logger.info("Running with browser-use agent (learning mode)...")
# Enable learning mode
self.is_learning = True
self.captured_steps = []
# Track LLM calls
original_get_model_output = self.agent.get_model_output
async def tracked_get_model_output(*args, **kwargs):
self.metrics['llm_calls'] += 1
return await original_get_model_output(*args, **kwargs)
self.agent.get_model_output = tracked_get_model_output
try:
# Run the agent
await self.agent.run(max_steps=max_steps)
# Check if task was successful
success = False
if self.agent.state.last_result:
for result in self.agent.state.last_result:
if result and hasattr(result, 'success') and result.success:
success = True
break
self.metrics['success'] = success
# If successful, save the workflow
if success and self.captured_steps:
await self._save_learned_workflow()
return {
'success': success,
'steps_completed': len(self.captured_steps),
'total_steps': len(self.captured_steps),
'execution_time': self.metrics['execution_time'],
'model_calls_saved': 0
}
finally:
self.is_learning = False
async def _save_learned_workflow(self):
"""Save the captured workflow to knowledge base."""
try:
# Create workflow from captured steps
workflow = Workflow(
workflow_id="", # Will be generated
intent=self.task,
description=f"Learned workflow for: {self.task}",
initial_url=self.captured_steps[0].get('url') if self.captured_steps else None
)
# Convert captured steps to workflow steps
for step_data in self.captured_steps:
step = WorkflowStep(
action_type=step_data['type'],
parameters=step_data['parameters']
)
# Add element info if available
if step_data.get('element_info'):
element_info = step_data['element_info']
step.xpath = element_info.get('xpath')
step.element_attributes = element_info.get('attributes', {})
workflow.add_step(step)
# Save to knowledge base
self.knowledge_base.save_workflow(workflow)
logger.info(f"Saved learned workflow with {len(workflow.steps)} steps")
except Exception as e:
logger.error(f"Failed to save learned workflow: {e}")
def _extract_task_parameters(self, task: str, workflow: Workflow) -> Dict[str, Any]:
"""
Extract parameters from task description for workflow.
Args:
task: Task description
workflow: Workflow that needs parameters
Returns:
Dictionary of extracted parameters
"""
# This is a simplified parameter extraction
# In production, you might use NLP or regex patterns
parameters = {}
# Example: Extract email addresses
import re
email_pattern = r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b'
emails = re.findall(email_pattern, task)
if emails:
parameters['recipient'] = emails[0]
# Extract quoted text as subject or content
quoted = re.findall(r'"([^"]*)"', task)
if quoted:
if 'subject' in task.lower() or '主题' in task.lower():
parameters['subject'] = quoted[0]
if len(quoted) > 1:
parameters['content'] = quoted[1]
else:
parameters['content'] = quoted[0]
return parameters
def run_sync(self, max_steps: int = 100) -> Dict[str, Any]:
"""
Synchronous wrapper for run method.
Args:
max_steps: Maximum steps for browser-use agent
Returns:
Execution results
"""
return asyncio.run(self.run(max_steps))