diff --git a/.github/workflows/darglint.yml b/.github/workflows/darglint.yml
index 133854e4..48bbf67a 100644
--- a/.github/workflows/darglint.yml
+++ b/.github/workflows/darglint.yml
@@ -31,4 +31,4 @@ jobs:
run: pip list
- name: Darglint checks
- run: darglint -v 2 -z short .
+ run: darglint -v 2 -z short src/
diff --git a/.gitignore b/.gitignore
index aa26dc9d..aa97d987 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,7 +3,7 @@ __pycache__/
*.py[cod]
*$py.class
results/
-.vscode
+
# C extensions
*.so
# Distribution / packaging
@@ -160,11 +160,14 @@ cython_debug/
# MacOS
**/.DS_Store
-.vscode
_sandbox.py
results/
# gradio
-.gradio/
\ No newline at end of file
+.gradio/
+
+outputs/
+miniwob-plusplus/
+.miniwob-server.pid
diff --git a/.vscode/launch.json b/.vscode/launch.json
new file mode 100644
index 00000000..6785b6d0
--- /dev/null
+++ b/.vscode/launch.json
@@ -0,0 +1,19 @@
+{
+ // Use IntelliSense to learn about possible attributes.
+ // Hover to view descriptions of existing attributes.
+ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+ "version": "0.2.0",
+ "configurations": [
+ {
+ "name": "Python Debugger: Current File",
+ "type": "debugpy",
+ "request": "launch",
+ "program": "${file}",
+ "console": "integratedTerminal",
+ "justMyCode": false,
+ "env": {
+ "AGENTLAB_DEBUG": "1"
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/.vscode/settings.json b/.vscode/settings.json
new file mode 100644
index 00000000..d52ef62d
--- /dev/null
+++ b/.vscode/settings.json
@@ -0,0 +1,15 @@
+{
+ "[python]": {
+ "editor.formatOnSave": true,
+ "editor.defaultFormatter": "ms-python.black-formatter",
+ "editor.codeActionsOnSave": {
+ "source.organizeImports": "explicit",
+ "source.fixAll": "never"
+ }
+ },
+ "python.testing.pytestArgs": [
+ "tests"
+ ],
+ "python.testing.unittestEnabled": false,
+ "python.testing.pytestEnabled": true,
+}
\ No newline at end of file
diff --git a/Makefile b/Makefile
new file mode 100644
index 00000000..b37fc1c4
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,32 @@
+.PHONY: test setup miniwob lint stop-miniwob
+
+setup:
+ @pip install -e .
+ @playwright install chromium --with-deps
+ @python -c 'import nltk; nltk.download("punkt_tab")'
+
+miniwob: stop-miniwob
+ @git clone https://github.com/Farama-Foundation/miniwob-plusplus.git || true
+ @cd miniwob-plusplus && git checkout 7fd85d71a4b60325c6585396ec4f48377d049838
+ @python -m http.server 8080 --directory miniwob-plusplus/miniwob/html & echo $$! > .miniwob-server.pid
+ @sleep 3
+ @echo "MiniWob server started on http://localhost:8080"
+
+check-miniwob:
+ @curl -I "http://localhost:8080/miniwob/" || (echo "MiniWob not reachable" && exit 1)
+ @echo "MiniWob server is reachable"
+
+stop-miniwob:
+ @kill -9 `cat .miniwob-server.pid` || true
+ @rm -f .miniwob-server.pid
+ @echo "MiniWob server stopped"
+
+run-tests:
+ @MINIWOB_URL="http://localhost:8080/miniwob/" pytest -n 5 --durations=10 -m 'not pricy' tests/
+ @echo "Tests completed"
+
+test: setup miniwob check-miniwob run-tests stop-miniwob
+
+lint: setup
+ @black src/ --check --diff
+ @darglint -v 2 -z short src/
diff --git a/requirements.txt b/requirements.txt
index c598b342..4b0f3d17 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,6 +5,7 @@ pytest==7.3.2
flaky
pytest-xdist
pytest-playwright
+pydantic~=2.9
dask
distributed
browsergym>=0.7.1
@@ -12,6 +13,7 @@ joblib>=1.2.0
openai>=1.7,<2
langchain_community
tiktoken
+tapeagents[converters]
huggingface_hub
contexttimer
ipython
@@ -24,3 +26,4 @@ matplotlib
ray[default]
python-slugify
pillow
+gymnasium>=0.27
\ No newline at end of file
diff --git a/src/agentlab/agents/README.md b/src/agentlab/agents/README.md
index 6af24e71..d65c0967 100644
--- a/src/agentlab/agents/README.md
+++ b/src/agentlab/agents/README.md
@@ -99,7 +99,7 @@ have to specify the type of each field (You can use Any if it is unknown)*
```python
from dataclasses import dataclass
from browsergym.experiment.agent import Agent
-from browsergym.experiment.loop import AgentArgs
+from agentlab.experiments.loop import AgentArgs
@dataclass
@@ -116,7 +116,7 @@ class CustomAgentArgs(AgentArgs):
To run experiments with your custom agent, define an instance of `ExpArgs` with the required parameters.
```python
-from browsergym.experiment.loop import ExpArgs
+from agentlab.experiments.loop import ExpArgs
exp_args = ExpArgs(
agent_args=CustomAgentArgs(custom_param="value"),
diff --git a/src/agentlab/agents/generic_agent/reproducibility_agent.py b/src/agentlab/agents/generic_agent/reproducibility_agent.py
index d9fa4c29..bf1f01c9 100644
--- a/src/agentlab/agents/generic_agent/reproducibility_agent.py
+++ b/src/agentlab/agents/generic_agent/reproducibility_agent.py
@@ -20,13 +20,10 @@
import bgym
from browsergym.experiments.agent import AgentInfo
-from browsergym.experiments.loop import ExpArgs, ExpResult, yield_all_exp_results
from bs4 import BeautifulSoup
-from langchain.schema import AIMessage, BaseMessage
-from langchain_community.adapters.openai import convert_message_to_dict
from agentlab.agents.agent_args import AgentArgs
-from agentlab.agents.dynamic_prompting import ActionFlags
+from agentlab.experiments.loop import ExpArgs, ExpResult, yield_all_exp_results
from agentlab.experiments.study import Study
from agentlab.llm.chat_api import make_assistant_message
from agentlab.llm.llm_utils import Discussion, messages_to_dict
@@ -65,7 +62,6 @@ def get_stats(self):
@dataclass
class ReproAgentArgs(GenericAgentArgs):
-
# starting with "_" will prevent from being part of the index in the load_results function
_repro_dir: str = None
@@ -81,7 +77,6 @@ def make_agent(self):
class ReproAgent(GenericAgent):
-
def __init__(
self,
chat_model_args,
@@ -93,7 +88,6 @@ def __init__(
super().__init__(chat_model_args, flags, max_retry)
def get_action(self, obs):
-
# replace the chat model with a reproducible chat that will mimic the
# same answers
step = len(self.actions)
diff --git a/src/agentlab/agents/most_basic_agent/most_basic_agent.py b/src/agentlab/agents/most_basic_agent/most_basic_agent.py
index 9da6d936..38145d5d 100644
--- a/src/agentlab/agents/most_basic_agent/most_basic_agent.py
+++ b/src/agentlab/agents/most_basic_agent/most_basic_agent.py
@@ -5,7 +5,7 @@
import bgym
from agentlab.agents.agent_args import AgentArgs
-from agentlab.llm.chat_api import make_system_message, make_user_message
+from agentlab.experiments.loop import ExpArgs
from agentlab.llm.llm_configs import CHAT_MODEL_ARGS_DICT
from agentlab.llm.llm_utils import (
Discussion,
@@ -133,7 +133,7 @@ def parser(response: str) -> tuple[dict, bool, str]:
# example for 2 experiments testing chain of thoughts on a miniwob task
exp_args = [
- bgym.ExpArgs(
+ ExpArgs(
agent_args=MostBasicAgentArgs(
temperature=0.1,
use_chain_of_thought=True,
@@ -142,7 +142,7 @@ def parser(response: str) -> tuple[dict, bool, str]:
env_args=env_args,
logging_level=logging.INFO,
),
- bgym.ExpArgs(
+ ExpArgs(
agent_args=MostBasicAgentArgs(
temperature=0.1,
use_chain_of_thought=False,
diff --git a/src/agentlab/agents/tapeagent/.gitignore b/src/agentlab/agents/tapeagent/.gitignore
deleted file mode 100644
index 7e78c9d4..00000000
--- a/src/agentlab/agents/tapeagent/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-TapeAgents/
-tapedata.sqlite
\ No newline at end of file
diff --git a/src/agentlab/agents/tapeagent/__init__.py b/src/agentlab/agents/tapeagent/__init__.py
index e69de29b..6c4130d4 100644
--- a/src/agentlab/agents/tapeagent/__init__.py
+++ b/src/agentlab/agents/tapeagent/__init__.py
@@ -0,0 +1,65 @@
+import json
+from dataclasses import asdict, is_dataclass
+
+import numpy as np
+from tapeagents.core import Step, StepMetadata
+from tapeagents.dialog_tape import AssistantStep, AssistantThought
+from tapeagents.io import save_json_tape, save_tape_images
+
+from agentlab.agents.tapeagent.agent import DictObservation, Tape, TapeAgent
+
+__all__ = ["as_tape", "save_tape", "TapeAgent", "Tape"]
+
+
+def as_tape(steps_info: list) -> Tape:
+ """
+ Create a Tape object from the steps info.
+
+ Args:
+ steps_info: list of StepInfo objects.
+
+ Returns:
+ Tape: a Tape object containing the steps and metadata.
+ """
+
+ class JsonEncoder(json.JSONEncoder):
+ def default(self, obj):
+ if is_dataclass(obj):
+ return asdict(obj) # type: ignore
+ if isinstance(obj, np.integer):
+ return int(obj)
+ if isinstance(obj, np.floating):
+ return float(obj)
+ if isinstance(obj, np.ndarray):
+ return obj.tolist()
+ return super().default(obj)
+
+ steps: list[Step] = []
+ for step_info in steps_info:
+ if step_info.obs is not None:
+ json_obs = json.dumps(step_info.obs, cls=JsonEncoder)
+ steps.append(DictObservation(content=json_obs))
+ if thought := step_info.agent_info.get("think"):
+ steps.append(AssistantThought(content=thought))
+ if step_info.action is not None:
+ step_metadata = StepMetadata(
+ other=dict(
+ reward=step_info.reward,
+ raw_reward=step_info.raw_reward,
+ terminated=step_info.terminated,
+ truncated=step_info.truncated,
+ agent_info=step_info.agent_info,
+ stats=step_info.stats,
+ )
+ )
+ steps.append(AssistantStep(content=step_info.action, metadata=step_metadata))
+ return Tape(steps=steps)
+
+
+def save_tape(exp_dir: str, episode_info: list, task: dict, tape: Tape):
+ tape.metadata.reward = sum([step.reward for step in episode_info])
+ tape.metadata.truncated = episode_info[-1].truncated
+ tape.metadata.terminated = episode_info[-1].terminated
+ tape.metadata.task = task
+ save_json_tape(tape, exp_dir, "tape.json")
+ save_tape_images(tape, f"{exp_dir}/tape_attachments")
diff --git a/src/agentlab/agents/tapeagent/agent.py b/src/agentlab/agents/tapeagent/agent.py
new file mode 100644
index 00000000..eefda1d1
--- /dev/null
+++ b/src/agentlab/agents/tapeagent/agent.py
@@ -0,0 +1,103 @@
+import logging
+from dataclasses import dataclass
+from typing import Literal
+
+import bgym
+import hydra
+from omegaconf import DictConfig
+from pydantic import Field
+from tapeagents.agent import Agent
+from tapeagents.core import Action, Observation, StopStep, TapeMetadata, Thought
+from tapeagents.core import Tape as BaseTape
+
+from agentlab.agents.agent_args import AgentArgs
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+
+class ExtendedMetadata(TapeMetadata):
+ name: str = ""
+ task: dict = {}
+ terminated: bool = False
+ truncated: bool = False
+ reward: float = 0.0
+ attempt_number: int = 0
+ other: dict = {}
+
+
+class Tape(BaseTape):
+ metadata: ExtendedMetadata = Field(default_factory=ExtendedMetadata) # type: ignore
+
+
+def load_config(config_name: str) -> DictConfig:
+ with hydra.initialize(config_path="conf", version_base="1.1"):
+ config = hydra.compose(config_name=config_name)
+ return config
+
+
+@dataclass
+class TapeAgentArgs(AgentArgs):
+ config: DictConfig = None # type: ignore
+
+ def make_agent(self) -> bgym.Agent:
+ agent: Agent = hydra.utils.instantiate(self.config.agent)
+ return TapeAgent(agent=agent)
+
+
+@dataclass
+class TapeAgentInfo(bgym.AgentInfo):
+ thoughts: list[Thought] = None # type: ignore
+
+
+class DictObservation(Observation):
+ """
+ Container for wrapping old dict observation into new Observation class.
+ """
+
+ kind: Literal["dict_observation"] = "dict_observation" # type: ignore
+ content: str
+
+
+class TapeAgent(bgym.Agent):
+ agent: Agent
+ tape: Tape
+
+ def __init__(self, agent: Agent):
+ super().__init__()
+ self.agent = agent
+ self.tape = Tape(steps=[])
+
+ def obs_preprocessor(self, obs: Observation | list[Observation]) -> list[Observation]:
+ if isinstance(obs, Observation):
+ obs = [obs]
+ assert isinstance(obs, list), f"Expected list of Observations, got {type(obs)}"
+ logger.info(f"Observations: {[type(o).__name__ for o in obs]}")
+ return obs
+
+ def get_action(self, obs: Observation | list[Observation]) -> tuple[Action, TapeAgentInfo]:
+ self.tape += obs # type: ignore
+ thoughts: list[Thought] = []
+ action = None
+ while not action:
+ for event in self.agent.run(self.tape):
+ if not event.step:
+ continue
+ self.tape = self.tape.append(event.step)
+ if isinstance(event.step, Thought):
+ thoughts.append(event.step)
+ logger.info(f"Thought: {event.step.llm_view()}")
+ elif isinstance(event.step, Action) and not action: # we use first action only
+ action = event.step
+ logger.info(f"Action: {action.llm_view()}")
+ else:
+ # there could be control flow steps for switching nodes and if clauses
+ logger.info(f"Other step: {type(event.step)}")
+ logger.info(f"Tape after run: ({len(self.tape)}) {[type(s).__name__ for s in self.tape]}")
+ return (action, TapeAgentInfo(thoughts=thoughts))
+
+ @property
+ def final_tape(self) -> Tape:
+ truncated = not any([isinstance(s, StopStep) for s in self.tape.steps])
+ self.tape.metadata = ExtendedMetadata(author=self.agent.name, truncated=truncated)
+ return self.tape
diff --git a/src/agentlab/agents/tapeagent/conf/agent/plan_act.yaml b/src/agentlab/agents/tapeagent/conf/agent/plan_act.yaml
new file mode 100644
index 00000000..017128cf
--- /dev/null
+++ b/src/agentlab/agents/tapeagent/conf/agent/plan_act.yaml
@@ -0,0 +1,94 @@
+_target_: tapeagents.agent.Agent
+name : gaia_agent
+max_iterations: 2
+llms:
+ default: ${llm}
+tools_description: |
+ - WebSearch - Performs a search in the web, wikipedia or youtube
+ - VideoReader - Opens video from a youtube URL. Can access the video content, thumbnail, subtitles and audio.
+ - Browser - Browser tool that can load web pages and interact with their content.
+ - CodeExecutor - Executes the python code snippet
+known_actions:
+ - _target_: hydra.utils.get_class
+ path: tapeagents.tools.web_search.SearchAction
+ - _target_: hydra.utils.get_class
+ path: tapeagents.steps.WatchVideoAction
+ - _target_: hydra.utils.get_class
+ path: tapeagents.tools.code_executor.PythonCodeAction
+ - _target_: hydra.utils.get_class
+ path: tapeagents.tools.browser.ClickAction
+ - _target_: hydra.utils.get_class
+ path: tapeagents.tools.browser.GoBackAction
+ - _target_: hydra.utils.get_class
+ path: tapeagents.tools.browser.GoForwardAction
+ - _target_: hydra.utils.get_class
+ path: tapeagents.tools.browser.OpenUrlAction
+ - _target_: hydra.utils.get_class
+ path: tapeagents.tools.simple_browser.PageDownAction
+ - _target_: hydra.utils.get_class
+ path: tapeagents.tools.simple_browser.PageUpAction
+
+templates:
+ system_prompt: |
+ You are an expert AI Agent trained to assist users with complex information processing tasks.
+ Your role is to understand user queries and respond in a helpful and accurate manner.
+ Keep your replies concise and direct. Prioritize clarity and avoid over-elaboration.
+ Do not express emotions or opinions about user questions.
+ allowed_tools: |
+ You have access to the following tools:
+ {tools_description}
+ thought_format: |
+ Important! Respond with the plain text, do not include any JSON or code.
+ Do not output anything besides what I asked in this message.
+ allowed_steps: |
+ You have access to the following tools:
+ {tools_description}
+ You are allowed to produce ONLY steps with the following JSON schemas:
+ {allowed_steps}
+ Do not reproduce the schema when producing steps; use it as a reference.
+ format: >
+ Output only a single JSON dict.
+ Do not repeat the last thought again.
+ If the last action does not change the observation, do not repeat it!
+ DO NOT OUTPUT ANYTHING BESIDES THE JSON! DO NOT PLACE ANY COMMENTS INSIDE THE JSON.
+ It will break the system that processes the output.
+
+nodes:
+ - _target_: tapeagents.nodes.StandardNode
+ name: plan
+ system_prompt: ${agent.templates.system_prompt}
+ guidance: |
+ Write a concise multi-step plan explaining which steps should be performed to find the answer for the given task.
+ Remember that you can use web search, browser, python code execution and access the youtube videos to reach your goals.
+ Be specific about how each step should be performed. Only describe the intended actions here, do not perform them yet.
+ Consider that next steps may depend on results of previous steps, so include conditional branching using "if" statements where needed.
+ ${agent.templates.thought_format}
+ steps_prompt: ${agent.templates.allowed_tools}
+
+ - _target_: tapeagents.nodes.StandardNode
+ name: facts_survey
+ system_prompt: ${agent.templates.system_prompt}
+ guidance: |
+ Before we begin executing the plan, please answer the following pre-survey.
+ Here is the pre-survey:
+ 1. Please list any specific facts or figures that are GIVEN in the request itself. It is possible that there are none.
+ 2. Please list any facts that may need to be looked up, and WHERE SPECIFICALLY they might be found. In some cases, authoritative sources are mentioned in the request itself.
+ 3. Please list any facts that may need to be derived (e.g., via logical deduction, simulation, or computation)
+ 4. Please list any facts that are recalled from memory, hunches, well-reasoned guesses, etc.
+ When answering this survey, keep in mind that "facts" will typically be specific names, dates, statistics, etc.
+ ${agent.templates.thought_format}
+ steps_prompt: ${agent.templates.allowed_tools}
+
+ - _target_: tapeagents.nodes.StandardNode
+ name: act
+ system_prompt: ${agent.templates.system_prompt}
+ guidance: |
+ Produce single next step. If the answer is ready, produce gaia_answer_action.
+ ${agent.templates.format}
+ steps_prompt: ${agent.templates.allowed_steps}
+ steps:
+ - tapeagents.steps.ReasoningThought
+ - agentlab.benchmarks.gaia.ExtractedFacts
+ - agentlab.benchmarks.gaia.GaiaAnswer
+ use_known_actions: true
+ next_node: act
\ No newline at end of file
diff --git a/src/agentlab/agents/tapeagent/conf/agent/plan_react.yaml b/src/agentlab/agents/tapeagent/conf/agent/plan_react.yaml
new file mode 100644
index 00000000..0d31c9c4
--- /dev/null
+++ b/src/agentlab/agents/tapeagent/conf/agent/plan_react.yaml
@@ -0,0 +1,102 @@
+_target_: tapeagents.agent.Agent
+name : gaia_agent
+max_iterations: 2
+llms:
+ default: ${llm}
+tools_description: |
+ - WebSearch - Performs a search in the web, wikipedia or youtube
+ - VideoReader - Opens video from a youtube URL. Can access the video content, thumbnail, subtitles and audio.
+ - Browser - Browser tool that can load web pages and interact with their content.
+ - CodeExecutor - Executes the python code snippet
+known_actions:
+ - _target_: hydra.utils.get_class
+ path: tapeagents.tools.web_search.SearchAction
+ - _target_: hydra.utils.get_class
+ path: tapeagents.steps.WatchVideoAction
+ - _target_: hydra.utils.get_class
+ path: tapeagents.tools.code_executor.PythonCodeAction
+ - _target_: hydra.utils.get_class
+ path: tapeagents.tools.browser.ClickAction
+ - _target_: hydra.utils.get_class
+ path: tapeagents.tools.browser.GoBackAction
+ - _target_: hydra.utils.get_class
+ path: tapeagents.tools.browser.GoForwardAction
+ - _target_: hydra.utils.get_class
+ path: tapeagents.tools.browser.OpenUrlAction
+ - _target_: hydra.utils.get_class
+ path: tapeagents.tools.simple_browser.PageDownAction
+ - _target_: hydra.utils.get_class
+ path: tapeagents.tools.simple_browser.PageUpAction
+
+templates:
+ system_prompt: |
+ You are an expert AI Agent trained to assist users with complex information processing tasks.
+ Your role is to understand user queries and respond in a helpful and accurate manner.
+ Keep your replies concise and direct. Prioritize clarity and avoid over-elaboration.
+ Do not express emotions or opinions about user questions.
+ allowed_tools: |
+ You have access to the following tools:
+ {tools_description}
+ thought_format: |
+ Important! Respond with the plain text, do not include any JSON or code.
+ Do not output anything besides what I asked in this message.
+ allowed_steps: |
+ You have access to the following tools:
+ {tools_description}
+ You are allowed to produce ONLY steps with the following JSON schemas:
+ {allowed_steps}
+ Do not reproduce the schema when producing steps; use it as a reference.
+ format: >
+ Output only a single JSON dict.
+ Do not repeat the last thought again.
+ If the last action does not change the observation, do not repeat it!
+ DO NOT OUTPUT ANYTHING BESIDES THE JSON! DO NOT PLACE ANY COMMENTS INSIDE THE JSON.
+ It will break the system that processes the output.
+
+nodes:
+ - _target_: tapeagents.nodes.StandardNode
+ name: plan
+ system_prompt: ${agent.templates.system_prompt}
+ guidance: |
+ Write a concise multi-step plan explaining which steps should be performed to find the answer for the given task.
+ Remember that you can use web search, browser, python code execution and access the youtube videos to reach your goals.
+ Be specific about how each step should be performed. Only describe the intended actions here, do not perform them yet.
+ Consider that next steps may depend on results of previous steps, so include conditional branching using "if" statements where needed.
+ ${agent.templates.thought_format}
+ steps_prompt: ${agent.templates.allowed_tools}
+
+ - _target_: tapeagents.nodes.StandardNode
+ name: facts_survey
+ system_prompt: ${agent.templates.system_prompt}
+ guidance: |
+ Before we begin executing the plan, please answer the following pre-survey.
+ Here is the pre-survey:
+ 1. Please list any specific facts or figures that are GIVEN in the request itself. It is possible that there are none.
+ 2. Please list any facts that may need to be looked up, and WHERE SPECIFICALLY they might be found. In some cases, authoritative sources are mentioned in the request itself.
+ 3. Please list any facts that may need to be derived (e.g., via logical deduction, simulation, or computation)
+ 4. Please list any facts that are recalled from memory, hunches, well-reasoned guesses, etc.
+ When answering this survey, keep in mind that "facts" will typically be specific names, dates, statistics, etc.
+ ${agent.templates.thought_format}
+ steps_prompt: ${agent.templates.allowed_tools}
+
+ - _target_: tapeagents.nodes.StandardNode
+ name: reflect
+ system_prompt: ${agent.templates.system_prompt}
+ guidance: |
+ Relect on last observation, after that propose the single next step.
+ ${agent.templates.thought_format}
+ steps_prompt: ${agent.templates.allowed_tools}
+
+ - _target_: tapeagents.nodes.StandardNode
+ name: act
+ system_prompt: ${agent.templates.system_prompt}
+ guidance: |
+ Produce single next step. If the answer is ready, produce gaia_answer_action.
+ ${agent.templates.format}
+ steps_prompt: ${agent.templates.allowed_steps}
+ steps:
+ - tapeagents.steps.ReasoningThought
+ - agentlab.benchmarks.gaia.ExtractedFacts
+ - agentlab.benchmarks.gaia.GaiaAnswer
+ use_known_actions: true
+ next_node: reflect
\ No newline at end of file
diff --git a/src/agentlab/agents/tapeagent/conf/environment/web_code.yaml b/src/agentlab/agents/tapeagent/conf/environment/web_code.yaml
new file mode 100644
index 00000000..f473283e
--- /dev/null
+++ b/src/agentlab/agents/tapeagent/conf/environment/web_code.yaml
@@ -0,0 +1,11 @@
+tools:
+ - _target_: tapeagents.tools.web_search.WebSearch
+ - _target_: tapeagents.tools.media_reader.VideoReader
+ exp_path: ""
+ - _target_: tapeagents.tools.browser.Browser
+ exp_path: ""
+ viewport_chars: 64000
+ navigation_only: true
+ - _target_: tapeagents.tools.code_executor.CodeExecutor
+ exp_path: ""
+ reuse_computer_container: true
\ No newline at end of file
diff --git a/src/agentlab/agents/tapeagent/conf/gaia_l1.yaml b/src/agentlab/agents/tapeagent/conf/gaia_l1.yaml
new file mode 100644
index 00000000..bbd2a11d
--- /dev/null
+++ b/src/agentlab/agents/tapeagent/conf/gaia_l1.yaml
@@ -0,0 +1,12 @@
+defaults:
+ - llm: o4mini
+ - agent: plan_act
+ - environment: web_code
+ - _self_
+
+name: gaia_agent
+comment: Gaia L1 val
+split: validation
+level: "1"
+parallel_backend: ray
+n_jobs: 10
\ No newline at end of file
diff --git a/src/agentlab/agents/tapeagent/conf/gaia_val.yaml b/src/agentlab/agents/tapeagent/conf/gaia_val.yaml
new file mode 100644
index 00000000..c867cecf
--- /dev/null
+++ b/src/agentlab/agents/tapeagent/conf/gaia_val.yaml
@@ -0,0 +1,12 @@
+defaults:
+ - llm: gpt4o_mini
+ - agent: plan_act
+ - environment: web_code
+ - _self_
+
+name: gaia_agent
+comment: Gaia val
+split: validation
+level: "all"
+parallel_backend: ray
+n_jobs: 10
\ No newline at end of file
diff --git a/src/agentlab/agents/tapeagent/conf/llm/gpt4o.yaml b/src/agentlab/agents/tapeagent/conf/llm/gpt4o.yaml
new file mode 100644
index 00000000..0fb74026
--- /dev/null
+++ b/src/agentlab/agents/tapeagent/conf/llm/gpt4o.yaml
@@ -0,0 +1,6 @@
+_target_: tapeagents.llms.LiteLLM
+model_name: gpt-4o-2024-08-06
+use_cache: true
+context_size: 128000
+parameters:
+ temperature: 0.2
\ No newline at end of file
diff --git a/src/agentlab/agents/tapeagent/conf/llm/gpt4o_mini.yaml b/src/agentlab/agents/tapeagent/conf/llm/gpt4o_mini.yaml
new file mode 100644
index 00000000..efc462cb
--- /dev/null
+++ b/src/agentlab/agents/tapeagent/conf/llm/gpt4o_mini.yaml
@@ -0,0 +1,6 @@
+_target_: tapeagents.llms.LiteLLM
+model_name: gpt-4o-mini-2024-07-18
+use_cache: true
+context_size: 128000
+parameters:
+ temperature: 0.2
\ No newline at end of file
diff --git a/src/agentlab/agents/tapeagent/conf/llm/o4mini.yaml b/src/agentlab/agents/tapeagent/conf/llm/o4mini.yaml
new file mode 100644
index 00000000..385ce3fc
--- /dev/null
+++ b/src/agentlab/agents/tapeagent/conf/llm/o4mini.yaml
@@ -0,0 +1,6 @@
+_target_: tapeagents.llms.LiteLLM
+model_name: o4-mini-2025-04-16
+use_cache: true
+context_size: 128000
+parameters:
+ temperature: 1.0
\ No newline at end of file
diff --git a/src/agentlab/agents/tapeagent/experiments/run_gaia.py b/src/agentlab/agents/tapeagent/experiments/run_gaia.py
new file mode 100644
index 00000000..ca613d46
--- /dev/null
+++ b/src/agentlab/agents/tapeagent/experiments/run_gaia.py
@@ -0,0 +1,25 @@
+import logging
+import os
+
+from agentlab.agents.tapeagent.agent import TapeAgentArgs, load_config
+from agentlab.benchmarks.gaia import GaiaBenchmark, stop_old_sandbox
+from agentlab.experiments.study import make_study
+
+fmt = "%(asctime)s - %(levelname)s - %(name)s:%(lineno)d - %(funcName)s() - %(message)s"
+logging.basicConfig(level=logging.INFO, force=True, format=fmt, handlers=[logging.StreamHandler()])
+
+if __name__ == "__main__":
+ config = load_config("gaia_l1")
+ study = make_study(
+ benchmark=GaiaBenchmark.from_config(config), # type: ignore
+ agent_args=TapeAgentArgs(agent_name=config.name, config=config),
+ comment=config.comment,
+ logging_level=logging.INFO,
+ logging_level_stdout=logging.INFO,
+ )
+ stop_old_sandbox()
+ if os.environ.get("AGENTLAB_DEBUG"):
+ study.exp_args_list = study.exp_args_list[:3]
+ study.run(n_jobs=1, n_relaunch=1, parallel_backend="sequential")
+ else:
+ study.run(n_jobs=config.n_jobs, n_relaunch=1, parallel_backend=config.parallel_backend)
diff --git a/src/agentlab/agents/tapeagent/experiments/setup_gaia.sh b/src/agentlab/agents/tapeagent/experiments/setup_gaia.sh
new file mode 100755
index 00000000..34d2dbc7
--- /dev/null
+++ b/src/agentlab/agents/tapeagent/experiments/setup_gaia.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# run podman for containers
+if ! command -v podman &> /dev/null; then
+ echo "Podman is not installed, installing..."
+ if ! command -v brew &> /dev/null; then
+ echo "Error: Homebrew is not installed. Please install it first."
+ echo "Visit https://brew.sh for installation instructions."
+ exit 1
+ fi
+ brew install podman
+ echo "Podman installed"
+ podman machine init > /dev/null 2>&1
+ echo "Podman initialized"
+fi
+if ! podman machine list | grep -q "Currently running"; then
+ podman machine set --user-mode-networking
+ nohup podman machine start > /dev/null 2>&1
+ echo "Podman machine started"
+ podman info > /dev/null 2>&1
+ if [ $? -ne 0 ]; then
+ echo "Error: Failed to initialize Podman. Please check the error messages above."
+ exit 1
+ fi
+fi
+export DOCKER_HOST=http+unix://$(podman machine inspect --format '{{.ConnectionInfo.PodmanSocket.Path}}')
+
+# Check if OPENAI_API_KEY is set
+if [ -z "${OPENAI_API_KEY}" ]; then
+ echo "Error: OPENAI_API_KEY environment variable is not set"
+ exit 1
+fi
+
+if [ -z "${SERPER_API_KEY}" ]; then
+ echo "Error: SERPER_API_KEY environment variable is not set"
+ exit 1
+fi
+
+# Run the Python script
+echo "You should be able to run the GAIA agent now using this command:"
+echo python "$(dirname "$0")/run_gaia.py"
\ No newline at end of file
diff --git a/src/agentlab/agents/tapeagent/install_tapeagents.sh b/src/agentlab/agents/tapeagent/install_tapeagents.sh
deleted file mode 100755
index bed32789..00000000
--- a/src/agentlab/agents/tapeagent/install_tapeagents.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/bash
-
-if [ ! -d "$(dirname "$0")/TapeAgents" ]; then
- # Clone the repository to this directory
- git clone https://github.com/ServiceNow/TapeAgents.git "$(dirname "$0")/TapeAgents"
- # Install the package in editable mode
- pip install -e "$(dirname "$0")/TapeAgents"
-else
- echo "TapeAgents directory already exists. Skipping installation."
-fi
diff --git a/src/agentlab/agents/tapeagent/main.py b/src/agentlab/agents/tapeagent/main.py
deleted file mode 100644
index fca5ba0e..00000000
--- a/src/agentlab/agents/tapeagent/main.py
+++ /dev/null
@@ -1,20 +0,0 @@
-from agentlab.agents.tapeagent.tapeagent import TapeAgentArgs
-from agentlab.experiments import study_generators
-from agentlab.llm.llm_configs import CHAT_MODEL_ARGS_DICT
-
-
-def main(benchmark: str, n_jobs: int, reproducibility: bool):
- agent_args = TapeAgentArgs(
- chat_model_args=CHAT_MODEL_ARGS_DICT["openai/gpt-4o-mini-2024-07-18"]
- )
- if reproducibility:
- agent_args.set_reproducibility_mode()
- study = study_generators.run_agents_on_benchmark(agent_args, benchmark)
- study.run(n_jobs=n_jobs, parallel_backend="joblib", strict_reproducibility=reproducibility)
- study.append_to_journal(strict_reproducibility=reproducibility)
-
-
-if __name__ == "__main__": # necessary for dask backend
- n_jobs = 8 # 1 when debugging in VSCode, -1 to use all available cores
- benchmark = "workarena.l1"
- main(benchmark, n_jobs, reproducibility=True)
diff --git a/src/agentlab/agents/tapeagent/tapeagent.py b/src/agentlab/agents/tapeagent/tapeagent.py
deleted file mode 100644
index e40672b5..00000000
--- a/src/agentlab/agents/tapeagent/tapeagent.py
+++ /dev/null
@@ -1,152 +0,0 @@
-import logging
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Any
-
-import bgym
-
-from agentlab.agents.agent_args import AgentArgs
-from agentlab.llm.chat_api import BaseModelArgs
-from agentlab.llm.tracking import cost_tracker_decorator
-
-##############################
-# TODO: replace this hacky imports after releasing tapeagents and tapeagents[examples] to pypi
-try:
- from tapeagents.llms import LiteLLM
- from tapeagents.tools.gym_browser import flatten_axtree
-except ImportError as e:
- print("Please run install_tapeagents.sh to install tapeagents first.")
- raise e
-
-import sys
-
-sys.path.append(str(Path(__file__).parent.resolve() / "TapeAgents"))
-##############################
-
-from examples.workarena.agent import WorkArenaAgent
-from examples.workarena.steps import (
- WorkArenaAction,
- ClickAction,
- GoBackAction,
- GoForwardAction,
- GotoPageAction,
- HoverAction,
- InputTextAction,
- PageObservation,
- PressAction,
- SelectOptionAction,
- ScrollAction,
- WorkArenaTape,
- WorkArenaTask,
- StopStep,
-)
-
-
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
-
-
-@dataclass
-class TapeAgentArgs(AgentArgs):
- agent_name: str = "WorkarenaTapeAgent"
- chat_model_args: BaseModelArgs = None
-
- def make_agent(self) -> bgym.Agent:
- llm = LiteLLM(
- model_name=self.chat_model_args.model_name,
- use_cache=False,
- context_size=self.chat_model_args.max_total_tokens,
- parameters={"temperature": self.chat_model_args.temperature},
- )
- return WorkarenaTapeAgent(llm)
-
- def set_reproducibility_mode(self):
- self.chat_model_args.temperature = 0
-
- def prepare(self):
- return self.chat_model_args.prepare_server()
-
- def close(self):
- return self.chat_model_args.close_server()
-
-
-class WorkarenaTapeAgent(bgym.Agent):
- tape: WorkArenaTape
-
- def __init__(self, llm: LiteLLM):
- self.tapeagent = WorkArenaAgent.create(llm)
- self.tape = WorkArenaTape()
-
- def obs_preprocessor(self, obs: dict) -> dict:
- axtree = obs.pop("axtree_object")
- obs["axtree_txt"] = flatten_axtree(axtree)
- return obs
-
- @cost_tracker_decorator
- def get_action(self, obs: Any) -> tuple[str, bgym.AgentInfo]:
- self.update_tape(obs)
- # run agent and collect thoughts and last action
- tape_segment = []
- action = None
- logger.info(f"Run tape with {len(self.tape)} steps")
- for event in self.tapeagent.run(self.tape):
- if not event.step:
- continue
- step = event.step
- tape_segment.append(step)
- logger.info(f"Generated step: {step.llm_view()}")
- if isinstance(step, WorkArenaAction):
- action = self.step_to_action(step)
- self.tape += tape_segment
-
- logger.info(f"Action string: {action}")
- return (
- action,
- bgym.AgentInfo(
- extra_info={"tape_segment": [step.model_dump() for step in tape_segment]},
- stats={},
- ),
- )
-
- def update_tape(self, obs: dict):
- """
- Update tape with new observation
- """
- obs_step = PageObservation(text=obs["axtree_txt"], current_page=1, total_pages=1)
- self.tape = self.tape.append(obs_step)
- if len(self.tape) == 1: # first observation
- logger.info("First observation, adding goal to tape")
- self.tape = self.tape.append(WorkArenaTask(task=obs["goal"]))
-
- def step_to_action(self, action: WorkArenaAction) -> str | None:
- """
- Convert action step to an action string with function call
- """
- action_str = ""
- if isinstance(action, GotoPageAction):
- action_str = f"goto('{action.url}')"
- elif isinstance(action, ClickAction):
- action_str = (
- f"click('{action.bid}', button='{action.button}', modifiers={action.modifiers})"
- )
- elif isinstance(action, SelectOptionAction):
- action_str = f"select_option('{action.bid}', '{action.option}')"
- elif isinstance(action, HoverAction):
- action_str = f"hover('{action.bid}')"
- elif isinstance(action, InputTextAction):
- text = action.text.replace("'", "\\'")
- action_str = f"fill('{action.bid}', '{text}')"
- elif isinstance(action, PressAction):
- f"press('{action.bid}', '{action.key_comb}')"
- elif isinstance(action, GoBackAction):
- action_str = "go_back()"
- elif isinstance(action, GoForwardAction):
- action_str = "go_forward()"
- elif isinstance(action, StopStep):
- logger.info("Stopping the loop")
- action_str = None
- elif isinstance(action, ScrollAction):
- action_str = "noop()" # TODO: implement scroll action
- else:
- raise ValueError(f"Unknown action type: {action}")
- return action_str
diff --git a/src/agentlab/analyze/agent_xray.py b/src/agentlab/analyze/agent_xray.py
index 9764898c..efac82f2 100644
--- a/src/agentlab/analyze/agent_xray.py
+++ b/src/agentlab/analyze/agent_xray.py
@@ -12,13 +12,13 @@
import numpy as np
import pandas as pd
from attr import dataclass
-from browsergym.experiments.loop import ExpResult, StepInfo
from langchain.schema import BaseMessage, HumanMessage
from openai import OpenAI
from PIL import Image
from agentlab.analyze import inspect_results
from agentlab.experiments.exp_utils import RESULTS_DIR
+from agentlab.experiments.loop import ExpResult, StepInfo
from agentlab.experiments.study import get_most_recent_study
from agentlab.llm.chat_api import make_system_message, make_user_message
from agentlab.llm.llm_utils import BaseMessage as AgentLabBaseMessage
@@ -201,7 +201,6 @@ def run_gradio(results_dir: Path):
"""
)
with gr.Row():
-
exp_dir_choice = gr.Dropdown(
choices=get_directory_contents(results_dir),
value=select_dir_instructions,
@@ -617,7 +616,7 @@ def update_logs():
try:
return f"""{info.exp_result.logs}"""
except FileNotFoundError:
- return f"""No Logs"""
+ return """No Logs"""
def update_stats():
@@ -757,11 +756,11 @@ def get_episode_info(info: Info):
info = f"""\
### {env_args.task_name} (seed: {env_args.task_seed})
-### Step {info.step} / {len(steps_info)-1} (Reward: {cum_reward:.1f})
+### Step {info.step} / {len(steps_info) - 1} (Reward: {cum_reward:.1f})
**Goal:**
-{code(str(AgentLabBaseMessage('', goal)))}
+{code(str(AgentLabBaseMessage("", goal)))}
**Task info:**
@@ -770,7 +769,7 @@ def get_episode_info(info: Info):
**exp_dir:**
{code(exp_dir_str)}"""
- except Exception as e:
+ except Exception:
info = f"""\
**Error while getting episode info**
{code(traceback.format_exc())}"""
@@ -942,7 +941,6 @@ def update_error_report():
def new_exp_dir(exp_dir, progress=gr.Progress(), just_refresh=False):
-
if exp_dir == select_dir_instructions:
return None, None
@@ -1075,7 +1073,6 @@ def add_patch(ax, start, stop, color, label, edge=False):
def plot_profiling(ax, step_info_list: list[StepInfo], summary_info: dict, progress_fn):
-
if len(step_info_list) == 0:
warning("No step info to plot")
return None
diff --git a/src/agentlab/analyze/inspect_results.py b/src/agentlab/analyze/inspect_results.py
index 41ee5c93..7d043fce 100644
--- a/src/agentlab/analyze/inspect_results.py
+++ b/src/agentlab/analyze/inspect_results.py
@@ -10,10 +10,11 @@
import numpy as np
import pandas as pd
-from browsergym.experiments.loop import ExpResult, get_exp_result, yield_all_exp_results
from IPython.display import display
from tqdm import tqdm
+from agentlab.experiments.loop import ExpResult, get_exp_result, yield_all_exp_results
+
# TODO find a more portable way to code set_task_category_as_index at least
# handle dynamic imports. We don't want to always import workarena
# from browsergym.workarena import TASK_CATEGORY_MAP
@@ -32,7 +33,11 @@ def get_constants_and_variables(df: pd.DataFrame, drop_constants: bool = False):
constants = {}
variable_keys = []
for col in df.columns:
- if df[col].nunique(dropna=False) == 1:
+ try:
+ nuniq = df[col].nunique(dropna=False)
+ except TypeError:
+ nuniq = 0 # non hashable types are considered variables
+ if nuniq == 1:
if isinstance(df[col].iloc[0], np.generic):
val = df[col].iloc[0].item()
else:
@@ -83,7 +88,7 @@ def set_index_from_variables(
white = any([fnmatch.fnmatch(var, pattern) for pattern in index_white_list])
black = any([fnmatch.fnmatch(var, pattern) for pattern in index_black_list])
- if white and (not black) and (not var in index_variables):
+ if white and (not black) and (var not in index_variables):
index_variables.append(var)
for var in index_variables:
@@ -205,7 +210,7 @@ def report_constant_and_variables(df, show_stack_traces=True):
if i >= 2:
break
if len(unique_counts) > 3:
- print(f" ...\n")
+ print(" ...\n")
def get_std_err(df, metric):
@@ -235,7 +240,7 @@ def get_sample_std_err(df, metric):
def summarize(sub_df):
- if not "cum_reward" in sub_df:
+ if "cum_reward" not in sub_df:
record = dict(
avg_reward=np.nan,
std_err=np.nan,
@@ -745,7 +750,7 @@ def summarize_study(result_df: pd.DataFrame) -> pd.DataFrame:
def split_by_key(df: pd.DataFrame, key):
"""Return a dict of dataframes spearted by the given key."""
# check if key in df
- if not (key in df.columns):
+ if key not in df.columns:
df = df.reset_index(key, inplace=False)
df_dict = {}
@@ -775,7 +780,7 @@ def get_all_summaries(results_dir: Path, skip_hidden=True, ignore_cache=False, i
summary.set_index("study_dir", inplace=True)
summaries.append(summary)
- except Exception as e:
+ except Exception:
traceback.print_exc()
continue
diff --git a/src/agentlab/analyze/tapes.py b/src/agentlab/analyze/tapes.py
new file mode 100644
index 00000000..3b1ef2bf
--- /dev/null
+++ b/src/agentlab/analyze/tapes.py
@@ -0,0 +1,241 @@
+import json
+import logging
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+import numpy as np
+import yaml
+from tapeagents.core import Step, StepMetadata
+from tapeagents.observe import retrieve_all_llm_calls
+from tapeagents.renderers.camera_ready_renderer import CameraReadyRenderer
+from tapeagents.tape_browser import TapeBrowser
+
+from agentlab.agents.tapeagent.agent import ExtendedMetadata, Tape
+from agentlab.benchmarks.gaia import step_error
+
+logger = logging.getLogger(__name__)
+fmt = "%(asctime)s - %(levelname)s - %(name)s:%(lineno)d - %(funcName)s() - %(message)s"
+logging.basicConfig(level=logging.INFO, force=True, format=fmt, handlers=[logging.StreamHandler()])
+
+
+class WrapperStep(Step):
+ content: dict
+
+
+def pretty_yaml(data: dict | None) -> str:
+ return yaml.dump(data, sort_keys=False, indent=2) if data else ""
+
+
+class TapesRender(CameraReadyRenderer):
+
+ @property
+ def style(self):
+ style = ""
+ return super().style + style
+
+ def render_step(self, step: WrapperStep, index: int, **kwargs):
+ step_dict = step.content.copy()
+ step_dict.pop("metadata", None)
+ kind = step_dict.pop("kind", "Step")
+ if kind == "set_next_node":
+ return ""
+ # remove empty keys
+ step_dict = {k: v for k, v in step_dict.items() if v is not None and v != ""}
+ if len(step_dict) == 1:
+ content = list(step_dict.values())[0]
+ elif kind == "page_observation":
+ content = step_dict.get("text", pretty_yaml(step_dict))
+ if len(content) > 100:
+ summary = content[:100]
+ content = f"{summary}
---
{content}Solved {acc:.2f}%, {n_solved} out of {len(tapes)}
"
+ if "all" in filename:
+ html += f"Prompt tokens: {prompt_tokens_num}
Output tokens: {output_tokens_num}
Cost: {total_cost:.2f} USDVisible
"
+ html += f"Prompt tokens: {visible_prompt_tokens_num}
Output tokens: {visible_output_tokens_num}
Cost: {visible_cost:.2f} USD"
+ html += f"Steps per tape: {avg_steps:.1f} ± {std_steps:.1f}
"
+ if errors:
+ errors_str = "
".join(f"{k}: {v}" for k, v in errors.items())
+ html += f"No result: {no_result}
"
+ html += f"Errors: {sum(errors.values())}
{errors_str}"
+ if actions:
+ actions_str = "
".join(f"{k}: {v}" for k, v in actions.items())
+ html += f"Actions: {sum(actions.values())}
{actions_str}"
+ if timers:
+ timers_str = "
".join(
+ f"{'execute ' if k.endswith('action') else ''}{k}: {v:.1f} sec, avg. {v/timer_counts[k]:.1f} sec"
+ for k, v in timers.items()
+ )
+ html += f"Timings
{timers_str}"
+ return html
+
+ def get_step_error(self, step_dict: dict, last_action: str | None) -> str:
+ return step_error(step_dict, last_action)
+
+ def calculate_accuracy(self, tapes: list[Tape]) -> tuple[float, int]:
+ solved = [tape.metadata.reward for tape in tapes]
+ accuracy = 100 * (sum(solved) / len(solved) if solved else 0.0)
+ return accuracy, int(sum(solved))
+
+ def aggregate_timer_times(self, tapes: list[Tape]):
+ timer_sums = defaultdict(float)
+ timer_counts = defaultdict(int)
+ for tape in tapes:
+ timers = tape.metadata.other.get("timers", {})
+ for timer_name, exec_time in timers.items():
+ timer_sums[timer_name] += exec_time
+ timer_counts[timer_name] += 1
+ for step in tape.steps:
+ action_kind = step.metadata.other.get("action_kind")
+ action_execution_time = step.metadata.other.get("action_execution_time")
+ if action_kind and action_execution_time:
+ timer_sums[action_kind] += action_execution_time
+ timer_counts[action_kind] += 1
+ return dict(timer_sums), dict(timer_counts)
+
+ def load_tapes(self, exp_dir: str) -> list[Tape]:
+ tapes: list[Tape] = []
+ fpath = Path(self.tapes_folder) / exp_dir
+ for json_file in fpath.rglob("tape.json"):
+ if json_file.stat().st_size == 0:
+ logger.warning(f"Empty tape file: {json_file}")
+ continue
+ try:
+ with open(json_file) as f:
+ tape_dict = json.load(f)
+ tape = Tape(steps=[], metadata=ExtendedMetadata(**tape_dict["metadata"]))
+ tape.steps = [
+ WrapperStep(content=s, metadata=StepMetadata(**s["metadata"]))
+ for s in tape_dict["steps"]
+ ]
+ tapes.append(tape)
+ except Exception as e:
+ logger.warning(f"Failed to load {json_file}: {e}")
+ logger.info(f"Loaded {len(tapes)} tapes from {fpath}")
+ self.exp_path = fpath
+ return sorted(
+ tapes,
+ key=lambda x: f"{x.metadata.task.get('Level', '')}{x.metadata.task.get('number', 0):03d}",
+ )
+
+ def save_annotation(self, step: int, annotation: str, tape_id: int):
+ pass
+
+
+if __name__ == "__main__":
+ results_dir = sys.argv[1] if len(sys.argv) > 1 else "~/agentlab_results/"
+ tapes_browser = TapesBrowser(Path(results_dir).expanduser())
+ tapes_browser.launch()
diff --git a/src/agentlab/benchmarks/abstract_env.py b/src/agentlab/benchmarks/abstract_env.py
new file mode 100644
index 00000000..33e09e22
--- /dev/null
+++ b/src/agentlab/benchmarks/abstract_env.py
@@ -0,0 +1,73 @@
+from abc import ABC, abstractmethod
+
+import gymnasium as gym
+from dataclasses_json import DataClassJsonMixin
+from pydantic import BaseModel
+
+
+class AbstractEnvArgs(DataClassJsonMixin):
+ @abstractmethod
+ def make_env(self, action_mapping, exp_dir, exp_task_kwargs) -> "AbstractEnv":
+ """Create an instance of the environment with the arguments stored in this object.
+
+ Args:
+ action_mapping (dict[str,str]): mapping from the agent's action space to the environment's action space
+ see AbstractActionSet.to_python_code from BrowserGym for an example
+ exp_dir (str): directory where the experiment is stored
+ exp_task_kwargs (dict[str,Any]): additional arguments for the environment
+
+ Returns:
+ env (AbstractEnv): instance of the environment.
+ """
+
+
+class AbstractBenchmark(BaseModel):
+ name: str
+ env_args_list: list
+
+ def get_version(self) -> int:
+ return "1"
+
+ def prepare_backends(self):
+ pass
+
+ def dependency_graph_over_tasks(self) -> dict[str, list[str]]:
+ return {}
+
+
+class AbstractEnv(gym.Env, ABC):
+ @abstractmethod
+ def reset(self, seed: int = None) -> tuple[dict[str, any], dict[str, any]]:
+ """Reset the environment to the initial state, ready for an agent to start a new episode.
+
+ Args:
+ seed (int): seed to be used for the environment's random number generator. Some task may
+ be deterministic and not require a seed.
+
+ Returns:
+ obs (dict[str,Any]): dictionary containing the observations
+ env_info (dict[str,Any]): additional information about the environment (see step's docstring)
+ """
+
+ @abstractmethod
+ def step(self, action: str):
+ """Exection action in the environment and return the next observations
+
+ Args:
+ action (str): action to be executed in the environment, as a string
+
+ Returns:
+ obs (dict[str,Any]): dictionary containing the observations
+ reward (float): reward obtained after executing the action
+ terminated (bool): whether the episode is terminated. The MDP reached a terminal state
+ truncated (bool): whether the episode is truncated. The episode was truncated due to external reasons
+ env_info (dict[str,Any]): additional information about the environment
+ task_info (str): Some potential debugging information about the task, not intended for the agent
+ action_exec_start (float): time when the action execution started
+ action_exec_stop (float): time when the action execution ended
+ action_exec_timeout (float): TODO I don't remember exactly what this is
+ """
+
+ @abstractmethod
+ def close(self):
+ """Close any resources used by the environment"""
diff --git a/src/agentlab/benchmarks/gaia.py b/src/agentlab/benchmarks/gaia.py
new file mode 100644
index 00000000..0468e305
--- /dev/null
+++ b/src/agentlab/benchmarks/gaia.py
@@ -0,0 +1,385 @@
+import logging
+import os
+import re
+import shutil
+import string
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Literal, Self
+
+import datasets
+import hydra
+import podman
+import tapeagents.config
+from omegaconf import DictConfig
+from pdf2image import convert_from_path
+from pydantic import ConfigDict, Field
+from tapeagents.core import Action, Observation, StopStep, Thought
+from tapeagents.environment import ContainerExecutor, StatefulTool, Tool
+from tapeagents.steps import ImageObservation
+from tapeagents.tools.simple_browser import SimpleTextBrowser
+
+from agentlab.benchmarks.abstract_env import AbstractBenchmark, AbstractEnvArgs
+from agentlab.benchmarks.multitool_gym import MultiToolGym
+
+logger = logging.getLogger(__name__)
+
+CONTAINER_NAME = "gaia_code_shared"
+
+
+class GaiaGym(MultiToolGym):
+ task: dict
+ exp_dir: str
+
+ def __init__(self, tools: list[Tool | StatefulTool], task: dict, exp_dir: str):
+ super().__init__(tools=tools)
+ self.task = task
+ self.exp_dir = exp_dir
+ os.makedirs(".cache", exist_ok=True)
+
+ def reset(self, seed=None) -> tuple[list[Observation], dict]:
+ """
+ Reset the state of all the tools and prepare initial observations from the task again
+ """
+ super().reset()
+ return task_to_observations(self.task), {}
+
+ def calculate_reward(self, action: Action) -> float:
+ if isinstance(action, GaiaAnswer):
+ model_answer = action.answer
+ ground_truth = self.task["Final answer"]
+ reward = 1.0 if question_scorer(model_answer, ground_truth) else 0.0
+ else:
+ reward = 0.0
+
+ if reward == 1.0:
+ logger.info(f"Task {self.task['task_id']} solved.")
+ else:
+ logger.info(f"Task {self.task['task_id']} failed.")
+
+ return reward
+
+
+@dataclass
+class GaiaGymArgs(AbstractEnvArgs):
+ model_config = ConfigDict(arbitrary_types_allowed=True)
+ task: dict[str, Any]
+ task_seed: int
+ task_name: str
+ env_config: DictConfig
+
+ def __init__(
+ self,
+ task_name: str,
+ task: dict[str, Any],
+ env_config: DictConfig,
+ task_seed: int = 0,
+ ):
+ self.task_name = task_name
+ self.task = task
+ self.task_seed = task_seed
+ self.env_config = env_config
+
+ def make_env(self, exp_dir: Path, action_mapping=None) -> GaiaGym:
+ tapeagents.config.DB_DEFAULT_FILENAME = str(exp_dir.parent / "tapedata.sqlite")
+ exp_dir_str = str(exp_dir)
+ logger.info(f"Init gaia env with directory {exp_dir_str}")
+ init_code_sandbox(exp_dir_str)
+ for i in range(len(self.env_config.tools)):
+ if hasattr(self.env_config.tools[i], "exp_path"):
+ self.env_config.tools[i].exp_path = exp_dir_str
+ tools = hydra.utils.instantiate(self.env_config.tools)
+ env = GaiaGym(tools=tools, task=self.task, exp_dir=exp_dir_str)
+ return env
+
+
+def init_code_sandbox(exp_dir: str) -> None:
+ # Use a common code directory for all tasks in the experiment, which is mounted in the container
+ root_exp_dir = Path(exp_dir).parent
+ code_path = os.path.join(root_exp_dir, "shared_code")
+ os.makedirs(code_path, exist_ok=True)
+ os.environ["COMPUTER_CONTAINER_NAME"] = CONTAINER_NAME
+
+ # symlink task code to the shared code directory
+ task_code_path = os.path.join(exp_dir, "code")
+ if not os.path.exists(task_code_path):
+ os.symlink(code_path, task_code_path)
+
+ try:
+ ContainerExecutor(container_name=CONTAINER_NAME, work_dir=code_path, no_deps=True)
+ except Exception as e:
+ logger.warning(f"Failed to initialize container executor: {e}")
+
+
+def stop_old_sandbox():
+ try:
+ podman.from_env().containers.get(CONTAINER_NAME).stop()
+ except Exception as e:
+ logger.warning(f"Failed to stop old container {CONTAINER_NAME}: {e}")
+
+
+class GaiaBenchmark(AbstractBenchmark):
+ model_config = ConfigDict(arbitrary_types_allowed=True)
+ name: str = "gaia"
+ split: Literal["test", "validation"]
+ level: Literal["1", "2", "3", "all"] = "all"
+ env_args_list: list[GaiaGymArgs] = None # type: ignore
+ dataset: dict | None = None # type: ignore
+ env_config: DictConfig = None # type: ignore
+
+ @classmethod
+ def from_config(cls, config: DictConfig, dataset: dict | None = None) -> Self:
+ return cls(
+ split=config.split,
+ level=config.level,
+ env_config=config.environment,
+ dataset=dataset,
+ )
+
+ def model_post_init(self, __context: Any) -> None:
+ self.env_args_list = []
+ number = 0
+ if self.dataset is None:
+ self.dataset = datasets.load_dataset(
+ path="gaia-benchmark/GAIA",
+ name="2023_all",
+ trust_remote_code=True,
+ ) # type: ignore
+ for task in self.dataset[self.split]: # type: ignore
+ if self.level != "all" and task["Level"] != self.level:
+ continue
+ number += 1
+ task["number"] = number
+ name = f"gaia.{task['task_id']}"
+ env_args = GaiaGymArgs(task_name=name, task=task, env_config=self.env_config)
+ self.env_args_list.append(env_args)
+ logger.info(f"Loaded {len(self.env_args_list)} tasks from {self.split} split")
+
+
+class ExtractedFacts(Thought):
+ """
+ Thought that contains the list of facts extracted from the document
+ """
+
+ kind: Literal["extracted_facts_thought"] = "extracted_facts_thought" # type: ignore
+ extracted_facts: list[str] | dict[str, Any] | str = Field(
+ description="facts extracted from the observation"
+ )
+
+
+class GaiaQuestion(Observation):
+ kind: Literal["question"] = "question" # type: ignore
+ content: str
+ filename: str | None = None
+
+ @classmethod
+ def from_task(cls, question: dict, files_dir: str = "/tmp/gaia_files"):
+ os.makedirs(files_dir, exist_ok=True)
+ question_prompt = question["Question"]
+ filename = None
+ if question["file_path"]:
+ basename = os.path.basename(question["file_path"])
+ tmp_fname = os.path.join(files_dir, basename)
+ shutil.copyfile(question["file_path"], tmp_fname)
+ assert os.path.exists(tmp_fname)
+ filename = tmp_fname
+ return cls(content=question_prompt, filename=filename)
+
+
+def task_to_observations(task: dict, max_doc_length: int = 8000) -> list[Observation]:
+ browser = SimpleTextBrowser()
+ question = GaiaQuestion.from_task(task)
+ if not question.filename:
+ return [question]
+
+ filename: str | None = question.filename
+ question.filename = None
+ steps: list[Observation] = []
+ name, ext = filename.rsplit(".", maxsplit=1)
+ ext = ext.lower()
+ if ext == "zip":
+ folder_name = name
+ os.makedirs(folder_name, exist_ok=True)
+ shutil.unpack_archive(filename, folder_name)
+ document_text = "\n\nArchive contains the following files:\n"
+ for i, file in enumerate(os.listdir(folder_name)):
+ file_path = os.path.join(folder_name, file)
+ content = browser.get_whole_document(file_path)
+ file_text = f"{i+1}. {file}. Content:\n{content}\n\n"
+ if len(file_text) > max_doc_length:
+ file_text = ""
+ file_text += f"{i+1}. Path to the '{file}': {file_path}"
+ document_text += file_text
+ elif ext in ("png", "jpg", "jpeg"):
+ steps.append(ImageObservation(image_path=filename, image_caption="Attached image"))
+ document_text = ""
+ else:
+ attach_doc_text = True
+ if ext == "pdf":
+ images, total_pages = pdf_to_images(filename)
+ if total_pages <= 3:
+ attach_doc_text = False
+ for i, img_path in enumerate(images):
+ steps.append(ImageObservation(image_path=img_path, image_caption=f"PDF page {i+1}"))
+ if attach_doc_text:
+ try:
+ content = browser.get_whole_document(filename)
+ except Exception as e:
+ logger.exception(f"Failed to read document: {e}")
+ content = ""
+ document_text = f"\n\nAttached {ext.upper()} file content:\n{content}\n"
+ if not len(content) or len(document_text) > max_doc_length:
+ document_text = ""
+ else:
+ document_text = "\nDocument pages attached as images below"
+ question.filename = filename
+ question.content += document_text
+ return [question] + steps
+
+
+def pdf_to_images(filename: str, n_pages: int = 3):
+ images = []
+ for i, image in enumerate(convert_from_path(filename)):
+ page_index = i + 1
+ page_fname = filename[:-4] + f"_{page_index}.png"
+ if os.path.exists(page_fname):
+ images.append(page_fname)
+ continue
+ image.save(page_fname)
+ images.append(page_fname)
+ return images[:n_pages], len(images)
+
+
+class GaiaAnswer(StopStep):
+ """
+ Action that indicates the agent has finished the plan and contains the answer or description of failure.
+ The answer should use already determined facts without additional conversion!
+ Your final answer should be a number OR as few words as possible OR a comma-separated list of numbers and/or strings.
+ ADDITIONALLY, your final answer MUST follow any formatting instructions specified in the original question (e.g., alphabetization, sequencing, units, rounding, decimal places, etc.)
+ If asked for a number, express it numerically, don't use commas, do not add anything after the number, don't include units such as $ or percent signs unless specified otherwise in the question.
+ If asked for a string, don't use articles or abbreviations (e.g. for cities), unless specified otherwise. Don't output any final sentence punctuation such as '.', '!', or '?'.
+ If asked for a comma-separated list, apply the above rules depending on whether the elements are numbers or strings.
+ If unable to determine the final answer, output an empty string.
+ """
+
+ kind: Literal["gaia_answer_action"] = "gaia_answer_action" # type: ignore
+ success: bool = Field(description="True if the task was successful, False otherwise")
+ overview: str = Field(
+ description="List of steps performed to answer the question. If the task was not successful, includes the reason for failure"
+ )
+ answer_unit: str = Field(
+ description="Unit of measurement for the answer, if applicable; otherwise an empty string"
+ )
+ answer: Any = Field(description="Short final answer")
+ long_answer: str = Field(description="Detailed final answer not restricted by format rules")
+
+
+def step_error(step_dict: dict, last_action: str | None) -> str:
+ kind = step_dict.get("kind", "unknown")
+ error = ""
+ if kind == "search_results_observation" and not len(step_dict.get("serp", [])):
+ error = "search_empty"
+ elif kind == "page_observation" and step_dict.get("error"):
+ error = "browser"
+ elif kind == "llm_output_parsing_failure_action":
+ error = "parsing"
+ elif kind == "action_execution_failure":
+ error = last_action if last_action else "action_failure"
+ elif kind == "code_execution_result" and step_dict.get("result", {}).get("exit_code"):
+ error = "code"
+ return error
+
+
+def normalize_number_str(number_str: str) -> float:
+ # we replace these common units and commas to allow
+ # conversion to float
+ for char in ["$", "%", ","]:
+ number_str = number_str.replace(char, "")
+ try:
+ return float(number_str)
+ except ValueError:
+ logger.info(f"String {number_str} cannot be normalized to number str.")
+ return float("inf")
+
+
+def split_string(
+ s: str,
+ char_list: list[str] = [",", ";"],
+) -> list[str]:
+ pattern = f"[{''.join(char_list)}]"
+ return re.split(pattern, s)
+
+
+def question_scorer(
+ model_answer: str,
+ ground_truth: str,
+) -> bool:
+ def is_float(element: Any) -> bool:
+ try:
+ float(element)
+ return True
+ except ValueError:
+ return False
+
+ # if gt is a number
+ if is_float(ground_truth):
+ logger.info(f"Evaluating {model_answer} as a number.")
+ normalized_answer = normalize_number_str(model_answer)
+ return normalized_answer == float(ground_truth)
+
+ # if gt is a list
+ elif any(char in ground_truth for char in [",", ";"]):
+ logger.info(f"Evaluating {model_answer} as a comma separated list.")
+ # question with the fish: normalization removes punct
+
+ gt_elems = split_string(ground_truth)
+ ma_elems = split_string(model_answer)
+
+ # check length is the same
+ if len(gt_elems) != len(ma_elems):
+ logger.warning("Answer lists have different lengths, returning False.")
+ return False
+
+ # compare each element as float or str
+ comparisons = []
+ for ma_elem, gt_elem in zip(ma_elems, gt_elems):
+ if is_float(gt_elem):
+ normalized_ma_elem = normalize_number_str(ma_elem)
+ comparisons.append(normalized_ma_elem == float(gt_elem))
+ else:
+ # we do not remove punct since comparisons can include punct
+ comparisons.append(
+ normalize_str(ma_elem, remove_punct=False)
+ == normalize_str(gt_elem, remove_punct=False)
+ )
+ return all(comparisons)
+
+ # if gt is a str
+ else:
+ logger.info(f"Evaluating {model_answer} as a string.")
+ return normalize_str(model_answer) == normalize_str(ground_truth)
+
+
+def normalize_str(input_str, remove_punct=True) -> str:
+ """
+ Normalize a string by:
+ - Removing all white spaces
+ - Optionally removing punctuation (if remove_punct is True)
+ - Converting to lowercase
+
+ Args:
+ input_str: str, the string to normalize
+ remove_punct: bool, whether to remove punctuation (default: True)
+
+ Returns:
+ str, the normalized string
+ """
+ # Remove all white spaces. Required e.g for seagull vs. sea gull
+ no_spaces = re.sub(r"\s", "", input_str)
+
+ # Remove punctuation, if specified.
+ if remove_punct:
+ translator = str.maketrans("", "", string.punctuation)
+ return no_spaces.lower().translate(translator)
+ else:
+ return no_spaces.lower()
diff --git a/src/agentlab/benchmarks/multitool_gym.py b/src/agentlab/benchmarks/multitool_gym.py
new file mode 100644
index 00000000..e91aa916
--- /dev/null
+++ b/src/agentlab/benchmarks/multitool_gym.py
@@ -0,0 +1,57 @@
+import logging
+import time
+
+from tapeagents.core import Action, Observation, StopStep
+from tapeagents.environment import ToolCollectionEnvironment
+from tapeagents.tools.base import StatefulTool, Tool
+
+from agentlab.benchmarks.abstract_env import AbstractEnv
+
+logger = logging.getLogger(__name__)
+
+
+class MultiToolGym(AbstractEnv):
+ def __init__(self, tools: list[Tool | StatefulTool], max_turns: int = 50):
+ self._env = ToolCollectionEnvironment(tools)
+ self._actions = self._env.actions()
+ self.max_turns = max_turns
+ self._turns = 0
+
+ def reset(self):
+ self._env.reset()
+ self._turns = 0
+
+ def step(self, action: Action) -> tuple[Observation, float, bool, bool, dict]:
+ logger.info(f"Gym {self.__class__.__name__} step called with action {type(action)}")
+ assert isinstance(action, Action)
+
+ action_exec_start = time.time()
+ terminated = isinstance(action, StopStep)
+ if terminated:
+ observation = Observation() # empty observation
+ else:
+ observation = self._env.step(action)
+ terminated = isinstance(observation, StopStep)
+ action_exec_stop = time.time()
+ self._turns += 1
+
+ reward = self.calculate_reward(action)
+
+ truncated = self._turns >= self.max_turns
+
+ env_info = {
+ "step_metadata": observation.metadata,
+ "action_exec_start": action_exec_start,
+ "action_exec_stop": action_exec_stop,
+ "action_exec_timeout": 0.0,
+ }
+ obs_view = observation.short_view() if isinstance(observation, Observation) else observation
+ logger.info(f"Gym {self.__class__.__name__} observation: {obs_view}")
+ return observation, reward, terminated, truncated, env_info
+
+ def calculate_reward(self, action: Action) -> float:
+ logger.warning("Reward calculation is not implemented, returning 0")
+ return 0.0
+
+ def close(self):
+ self._env.close()
diff --git a/src/agentlab/experiments/exp_utils.py b/src/agentlab/experiments/exp_utils.py
index 5759e6d1..27b909b2 100644
--- a/src/agentlab/experiments/exp_utils.py
+++ b/src/agentlab/experiments/exp_utils.py
@@ -6,9 +6,10 @@
from pathlib import Path
from time import sleep, time
-from browsergym.experiments.loop import ExpArgs, yield_all_exp_results
from tqdm import tqdm
+from agentlab.experiments.loop import ExpArgs, yield_all_exp_results
+
logger = logging.getLogger(__name__) # Get logger based on module name
@@ -63,7 +64,6 @@ def timeout_manager(seconds: int = None):
return
def alarm_handler(signum, frame):
-
logger.warning(f"Operation timed out after {seconds}s, raising TimeoutError.")
# send sigint
# os.kill(os.getpid(), signal.SIGINT) # this doesn't seem to do much I don't know why
@@ -176,11 +176,11 @@ def hide_some_exp(base_dir, filter: callable, just_test):
msg = f"Searching {len(exp_list)} experiments to move to _* expriments where `filter(exp_args)` is True."
if just_test:
- msg += f"\nNote: This is a just a test, no experiments will be moved. Set `just_test=False` to move them."
+ msg += "\nNote: This is a just a test, no experiments will be moved. Set `just_test=False` to move them."
logging.info(msg)
- exp_list = tqdm(exp_list, desc=f"Filtering experiments.")
+ exp_list = tqdm(exp_list, desc="Filtering experiments.")
filtered_out = []
for exp in exp_list:
diff --git a/src/agentlab/experiments/launch_exp.py b/src/agentlab/experiments/launch_exp.py
index 962115c2..3bc6c54e 100644
--- a/src/agentlab/experiments/launch_exp.py
+++ b/src/agentlab/experiments/launch_exp.py
@@ -3,9 +3,9 @@
from pathlib import Path
import bgym
-from browsergym.experiments.loop import ExpArgs, yield_all_exp_results
from agentlab.experiments.exp_utils import run_exp
+from agentlab.experiments.loop import ExpArgs, yield_all_exp_results
def run_experiments(
@@ -98,7 +98,7 @@ def run_experiments(
logging.info("All jobs are finished. Calling agent_args.close() on all agents...")
for exp_args in exp_args_list:
exp_args.agent_args.close()
- logging.info("Experiment finished.")
+ logging.info(f"Experiment finished and saved in {study_dir}.")
def find_incomplete(study_dir: str | Path, include_errors=True):
@@ -142,8 +142,8 @@ def find_incomplete(study_dir: str | Path, include_errors=True):
else:
logging.info(f"Found {job_count} incomplete experiments in {study_dir}.")
- message = f"Make sure the processes that were running are all stopped. Otherwise, "
- f"there will be concurrent writing in the same directories.\n"
+ message = "Make sure the processes that were running are all stopped. Otherwise, "
+ "there will be concurrent writing in the same directories.\n"
logging.info(message)
diff --git a/src/agentlab/experiments/loop.py b/src/agentlab/experiments/loop.py
new file mode 100644
index 00000000..5a9580ca
--- /dev/null
+++ b/src/agentlab/experiments/loop.py
@@ -0,0 +1,922 @@
+import gzip
+import importlib.metadata
+import json
+import logging
+import os
+import pickle
+import re
+import sys
+import time
+import traceback
+import uuid
+from abc import ABC, abstractmethod
+from collections import defaultdict
+from dataclasses import asdict, dataclass, field, is_dataclass
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+
+import gymnasium as gym
+import numpy as np
+from browsergym.core.chat import Chat
+from browsergym.experiments.agent import Agent
+from browsergym.experiments.utils import count_messages_token, count_tokens
+from dataclasses_json import DataClassJsonMixin
+from PIL import Image
+from tqdm import tqdm
+
+from agentlab.agents.tapeagent import TapeAgent, save_tape
+
+logger = logging.getLogger(__name__)
+
+SEED_MAX = 2 ^ 32 # arbitrary max value (exclusive), seems large enough
+
+
+@dataclass
+class EnvArgs(DataClassJsonMixin):
+ task_name: str
+ task_seed: Optional[int] = None
+ max_steps: Optional[int] = None
+ headless: bool = True
+ record_video: bool = False
+ wait_for_user_message: bool = False
+ viewport: Optional[dict] = None # use default value from BrowserGym
+ slow_mo: Optional[int] = None # use default value from BrowserGym
+ storage_state: Optional[str | Path | dict] = None
+ task_kwargs: Optional[dict] = None # use default value from BrowserGym
+
+ def make_env(self, action_mapping, exp_dir, exp_task_kwargs: dict = {}):
+ """
+ Instantiates the BrowserGym environment corresponding to the arguments (with some tweaks).
+
+ Args:
+ action_mapping: overrides the action mapping of the environment.
+ exp_dir: will set some environment parameters (e.g., record_video_dir) with respect to the directory where the experiment is running.
+ exp_task_kwargs: use with caution! Will override task parameters to experiment-specific values. Useful to set different server configs for different experiments, or output file paths within the experiment's folder (e.g., assistantbench).
+
+ Returns:
+ env: the gym environment.
+ """
+ extra_kwargs = {}
+ if self.record_video:
+ extra_kwargs["record_video_dir"] = exp_dir
+ if self.viewport:
+ extra_kwargs["viewport"] = self.viewport
+ if self.slow_mo is not None:
+ extra_kwargs["slow_mo"] = self.slow_mo
+ if self.storage_state:
+ extra_kwargs["pw_context_kwargs"] = {"storage_state": self.storage_state}
+ if self.task_kwargs is not None:
+ extra_kwargs["task_kwargs"] = self.task_kwargs
+ if exp_task_kwargs:
+ extra_kwargs["task_kwargs"] = extra_kwargs.get("task_kwargs", {}) | exp_task_kwargs
+
+ # assistantbench hack, write the task output (agent prediction) to a file in the experiment's directory
+ # TODO: find a better way to deal with this
+ if self.task_name.startswith("assistantbench.test"):
+ extra_kwargs["task_kwargs"] = extra_kwargs.get("task_kwargs", {}) | {
+ "output_file": exp_dir / "assistantbench-prediction.json"
+ }
+
+ return gym.make(
+ _get_env_name(self.task_name),
+ disable_env_checker=True,
+ max_episode_steps=self.max_steps,
+ headless=self.headless,
+ wait_for_user_message=self.wait_for_user_message,
+ action_mapping=action_mapping, # action mapping is provided by the agent
+ **extra_kwargs,
+ )
+
+
+@dataclass
+class AbstractAgentArgs(ABC):
+ """A template class that defines the required signature of an agent's arguments."""
+
+ agent_name: str = None # type: ignore
+
+ def __post_init__(self):
+ if self.agent_name is None:
+ self.agent_name = self.__class__.__name__
+
+ def prepare(self):
+ """Prepare the agent's LLM models before running the experiment."""
+ pass
+
+ def close(self):
+ """Close the agent's LLM models after running the experiment."""
+ pass
+
+ @abstractmethod
+ def make_agent(self) -> Agent:
+ """Comply the experiments.loop API for instantiating the agent."""
+
+
+def save_package_versions(exp_dir: Path):
+ """Save the versions of the installed packages in the experiment directory."""
+ python_dists = "\n".join(
+ sorted(
+ [
+ f"{dist.metadata['Name']}=={dist.metadata['Version']}"
+ for dist in importlib.metadata.distributions()
+ ]
+ )
+ )
+ (exp_dir / "package_versions.txt").write_text(python_dists)
+
+
+@dataclass
+class StepTimestamps:
+ env_start: float = 0
+ action_exec_start: float = 0 # to extract begining of visual action from video
+ action_exec_stop: float = 0 # to extract end of visual action from video
+ action_exect_after_timeout: float = 0
+ env_stop: float = 0
+ agent_start: float = 0
+ agent_stop: float = 0
+
+
+@dataclass
+class StepInfo:
+ """Collects information about step that will be saved and reloaded.
+ Helper functions only modify the dataclass attributes and helps keeping the
+ information organized.
+
+ Attributes:
+ -----------
+ step: int
+ The step number of the episode.
+ obs: dict
+ The observation of the environment.
+ reward: float
+ The reward of the step.
+ raw_reward: float
+ The raw reward of the step.
+ terminated: bool
+ Whether the episode is terminated i.e. reached a terminal state.
+ truncated: bool
+ Whether the episode is truncated i.e. reached a maximum number of steps.
+ action: str
+ The action taken by the agent.
+ agent_info: dict
+ Additional information from the agent.
+ stats: dict
+ Extra statistics about the step.
+ profiling: StepTimestamps
+ Timestamps of the different events during the episode.
+ """
+
+ step: int = None
+ obs: dict = None
+ reward: float = 0
+ raw_reward: float = 0
+ terminated: bool = None
+ truncated: bool = None
+ action: str = None
+ agent_info: dict = field(default_factory=dict)
+ stats: dict = None
+ profiling: StepTimestamps = field(default_factory=StepTimestamps)
+ task_info: dict = None
+
+ def from_step(self, env: gym.Env, action: str, obs_preprocessor: callable):
+ t = self.profiling
+ t.env_start = time.time()
+ self.obs, self.reward, self.terminated, self.truncated, env_info = env.step(action)
+ t.env_stop = time.time()
+
+ self.task_info = env_info.get("task_info", None)
+
+ self.raw_reward = env_info.get("RAW_REWARD_GLOBAL", None)
+
+ t.action_exec_start = env_info["action_exec_start"] # start
+ t.action_exect_after_timeout = env_info["action_exec_stop"]
+ t.action_exec_stop = env_info["action_exec_stop"] - env_info["action_exec_timeout"]
+
+ if obs_preprocessor:
+ self.obs = obs_preprocessor(self.obs)
+
+ def from_action(self, agent: Agent):
+ self.profiling.agent_start = time.time()
+ self.action, self.agent_info = agent.get_action(self.obs.copy())
+ self.profiling.agent_stop = time.time()
+
+ self.make_stats()
+
+ return self.action
+
+ def from_reset(self, env: gym.Env, seed: int, obs_preprocessor: callable):
+ t = self.profiling
+ t.env_start = time.time()
+ self.obs, env_info = env.reset(seed=seed)
+ self.reward, self.terminated, self.truncated = 0, False, False
+ t.env_stop = time.time()
+
+ t.action_exec_start = env_info.get("recording_start_time", t.env_start)
+ t.action_exect_after_timeout = t.env_stop
+ t.action_exec_stop = t.env_stop
+
+ if obs_preprocessor:
+ self.obs = obs_preprocessor(self.obs)
+
+ @property
+ def is_done(self):
+ return self.terminated or self.truncated
+
+ def make_stats(self):
+ if isinstance(self.obs, dict):
+ stats = {
+ f"n_token_{key}": count_tokens(val)
+ for key, val in self.obs.items()
+ if isinstance(val, str)
+ }
+ else:
+ stats = {}
+ stats.update(self.agent_info.pop("stats", {}))
+
+ messages = self.agent_info.get("chat_messages", None)
+ if messages is not None:
+ stats["n_token_agent_messages"] = count_messages_token(messages)
+
+ t = self.profiling
+ stats["step_elapsed"] = t.env_stop - t.env_start
+ stats["agent_elapsed"] = t.agent_stop - t.agent_start
+
+ self.stats = stats
+
+ def save_step_info(self, exp_dir, save_json=False, save_screenshot=True, save_som=False):
+ # special treatment for some of the observation fields
+ if isinstance(self.obs, dict):
+ # save screenshots to separate files
+ screenshot = self.obs.pop("screenshot", None)
+ screenshot_som = self.obs.pop("screenshot_som", None)
+
+ if save_screenshot and screenshot is not None:
+ img = Image.fromarray(screenshot)
+ img.save(exp_dir / f"screenshot_step_{self.step}.png")
+
+ if save_som and screenshot_som is not None:
+ img = Image.fromarray(screenshot_som)
+ img.save(exp_dir / f"screenshot_som_step_{self.step}.png")
+
+ # save goal object (which might contain images) to a separate file to save space
+ if self.obs.get("goal_object", False):
+ # save the goal object only once (goal should never change once setup)
+ goal_object_file = Path(exp_dir) / "goal_object.pkl.gz"
+ if not goal_object_file.exists():
+ with gzip.open(goal_object_file, "wb") as f:
+ pickle.dump(self.obs["goal_object"], f)
+ # set goal_object to a special placeholder value, which indicates it should be loaded from a separate file
+ self.obs["goal_object"] = None
+
+ with gzip.open(exp_dir / f"step_{self.step}.pkl.gz", "wb") as f:
+ pickle.dump(self, f)
+
+ if save_json:
+ with open(exp_dir / "steps_info.json", "w") as f:
+ json.dump(self, f, indent=4, cls=DataclassJSONEncoder)
+
+ if isinstance(self.obs, dict):
+ # add the screenshots back to the obs
+ # why do we need this?
+ if screenshot is not None:
+ self.obs["screenshot"] = screenshot
+ if screenshot_som is not None:
+ self.obs["screenshot_som"] = screenshot_som
+
+
+@dataclass
+class ExpArgs:
+ """Arguments to run an experiment, i.e. run agent in an environment until done.
+
+ This dataclass is used to store experiments arguments. It contains
+ agent_args and env_args which follows the same principle. It contains helper
+ functions to prepare and run experiments.
+
+ Attributes:
+ -----------
+ agent_args: AbstractAgentArgs
+ The arguments to instantiate the agent.
+ env_args: EnvArgs
+ The arguments to instantiate the environment.
+ exp_dir: str
+ The directory where the experiment will be saved.
+ exp_name: str
+ The name of the experiment. If None, it will be generated from the
+ agent and environment names.
+ enable_debug: bool
+ If python is running in debug mode and `enable_debug` is True, errors
+ will be raised instead of only logged
+ error_msg: str
+ Error that occured while running the experiment (if any).
+ stack_trace: str
+ Stack trace of the error (if any).
+ order: int (internal)
+ The order of the experiment in the batch. It is used to keep track of
+ the original order of the experiments in case they are shuffled.
+ """
+
+ agent_args: AbstractAgentArgs
+ env_args: EnvArgs
+ exp_dir: str = None
+ exp_name: str = None
+ enable_debug: bool = True
+ err_msg: str = None
+ stack_trace: str = None
+ order: int = None # use to keep the original order the experiments were meant to be launched.
+ logging_level: int = logging.INFO
+ logging_level_stdout: int = logging.INFO
+ exp_id: str = None
+ depends_on: tuple[str] = ()
+ save_screenshot: bool = True
+ save_som: bool = False
+
+ def make_id(self):
+ """Create a unique id for the experiment."""
+ if self.exp_id is None:
+ self.exp_id = str(uuid.uuid4())
+
+ def prepare(self, exp_root):
+ """Prepare the experiment directory and save the experiment arguments.
+
+ This enables inspecting experiments that are not run yet.
+
+ Args:
+ exp_root: str
+ The root directory where the experiment will be saved.
+ """
+ if self.env_args.task_seed is None:
+ self.env_args.task_seed = np.random.randint(0, SEED_MAX)
+
+ if self.exp_name is None:
+ task_name = self.env_args.task_name
+ self.exp_name = f"{self.agent_args.agent_name}_on_{task_name}_{self.env_args.task_seed}"
+
+ # if exp_dir exists, it means it's a re-run, move the old one
+ if self.exp_dir is not None:
+ _move_old_exp(self.exp_dir)
+
+ self.make_id()
+
+ self.exp_date = datetime.now()
+ self._make_dir(exp_root)
+
+ self.exp_dir.mkdir(parents=True, exist_ok=True)
+ with open(self.exp_dir / "exp_args.pkl", "wb") as f:
+ pickle.dump(self, f)
+
+ def _make_dir(self, exp_root):
+ """Create a unique directory for the experiment."""
+ date_str = self.exp_date.strftime("%Y-%m-%d_%H-%M-%S")
+ exp_str = re.sub(
+ r"[\/:*?<>|]", "_", self.exp_name
+ ) # sanitize exp_name to be used as a file name (substitute forbidden characters)
+
+ for i in range(1000):
+ if i >= 999: # make sure we don't loop forever
+ raise ValueError("Could not find a unique name for the experiment directory.")
+
+ tag = f"_{i}" if i > 0 else ""
+ self.exp_dir = Path(exp_root) / f"{date_str}_{exp_str}{tag}"
+ if not self.exp_dir.exists():
+ break
+
+ # TODO distinguish between agent error and environment or system error. e.g.
+ # the parsing error of an action should not be re-run.
+ def run(self):
+ """Run the experiment and save the results"""
+ # start writing logs to run logfile
+ self._set_logger()
+
+ # log python environment info
+ save_package_versions(Path(self.exp_dir))
+
+ episode_info = []
+ agent = None
+ env, step_info, err_msg, stack_trace = None, None, None, None
+ try:
+ logger.info(f"Running experiment {self.exp_name} in:\n {self.exp_dir}")
+ agent = self.agent_args.make_agent()
+ logger.debug("Agent created.")
+
+ env = self.env_args.make_env(
+ action_mapping=agent.action_set.to_python_code,
+ exp_dir=self.exp_dir,
+ )
+
+ logger.debug("Environment created.")
+ step_info = StepInfo(step=0)
+ episode_info = [step_info]
+ step_info.from_reset(
+ env, seed=self.env_args.task_seed or 0, obs_preprocessor=agent.obs_preprocessor
+ )
+ logger.debug("Environment reset.")
+
+ while not step_info.is_done: # set a limit
+ logger.debug(f"Starting step {step_info.step}.")
+ action = step_info.from_action(agent)
+ logger.debug(f"Agent chose action:\n {action}")
+
+ if action is None:
+ # will end the episode after saving the step info.
+ step_info.truncated = True
+
+ step_info.save_step_info(
+ self.exp_dir, save_screenshot=self.save_screenshot, save_som=self.save_som
+ )
+ logger.debug("Step info saved.")
+
+ if hasattr(env.unwrapped, "chat") and isinstance(env.unwrapped.chat, Chat):
+ _send_chat_info(env.unwrapped.chat, action, step_info.agent_info)
+ logger.debug("Chat info sent.")
+
+ if action is None:
+ logger.debug("Agent returned None action. Ending episode.")
+ break
+
+ step_info = StepInfo(step=step_info.step + 1)
+ episode_info.append(step_info)
+
+ logger.debug("Sending action to environment.")
+ step_info.from_step(env, action, obs_preprocessor=agent.obs_preprocessor)
+ logger.debug("Environment stepped.")
+
+ except Exception as e:
+ err_msg = f"Exception uncaught by agent or environment in task {self.env_args.task_name}.\n{type(e).__name__}:\n{e}"
+ stack_trace = traceback.format_exc()
+
+ self.err_msg = err_msg
+ self.stack_trace = stack_trace
+
+ logger.warning(err_msg + "\n" + stack_trace)
+ if _is_debugging() and self.enable_debug:
+ logger.warning("Debug mode is enabled. Raising the error.")
+ raise
+
+ finally:
+ try:
+ if step_info is not None:
+ step_info.save_step_info(
+ self.exp_dir, save_screenshot=self.save_screenshot, save_som=self.save_som
+ )
+ except Exception as e:
+ logger.error(f"Error while saving step info in the finally block: {e}")
+ try:
+ if (
+ not err_msg
+ and len(episode_info) > 0
+ and not (episode_info[-1].terminated or episode_info[-1].truncated)
+ ):
+ e = KeyboardInterrupt("Early termination??")
+ err_msg = f"Exception uncaught by agent or environment in task {self.env_args.task_name}.\n{type(e).__name__}:\n{e}"
+ logger.info("Saving experiment info.")
+ self.save_summary_info(episode_info, Path(self.exp_dir), err_msg, stack_trace)
+ if isinstance(agent, TapeAgent):
+ task = getattr(env, "task", {})
+ save_tape(self.exp_dir, episode_info, task, agent.final_tape)
+ except Exception as e:
+ logger.exception(f"Error while saving experiment info: {e}")
+ try:
+ if env is not None:
+ env.close()
+ except Exception as e:
+ logger.exception(f"Error while closing the environment: {e}")
+ try:
+ self._unset_logger() # stop writing logs to run logfile
+ except Exception as e:
+ logger.exception(f"Error while unsetting the logger: {e}")
+
+ def _set_logger(self):
+ # output logging traces to a log file
+ file_handler = logging.FileHandler(self.exp_dir / "experiment.log")
+ file_handler.setLevel(self.logging_level) # same level as console outputs
+ formatter = logging.Formatter(
+ "%(asctime)s - %(process)d - %(name)s - %(levelname)s - %(message)s"
+ )
+ file_handler.setFormatter(formatter)
+ # output handler
+ stream_handler = logging.StreamHandler()
+ stream_handler.setLevel(self.logging_level_stdout)
+ stream_handler.setFormatter(formatter)
+ # setup root logger
+ root_logger = logging.getLogger()
+
+ # remove previous stream handlers
+ for handler in root_logger.handlers:
+ if isinstance(handler, logging.StreamHandler):
+ root_logger.removeHandler(handler)
+
+ root_logger.setLevel(self.logging_level)
+ root_logger.addHandler(file_handler)
+ root_logger.addHandler(stream_handler)
+ # setup openai logger (don't go below INFO verbosity)
+ openai_logger = logging.getLogger("openai._base_client")
+ openai_logger.setLevel(max(logging.INFO, self.logging_level))
+
+ self.logging_file_handler = file_handler
+
+ def _unset_logger(self):
+ root_logger = logging.getLogger()
+ root_logger.removeHandler(self.logging_file_handler)
+
+ def save_summary_info(
+ self,
+ episode_info: list[StepInfo],
+ exp_dir: Path,
+ err_msg: str | None,
+ stack_trace: str | None,
+ ):
+ # bring err from agent_info to the top level
+ if err_msg is None:
+ err_msg, stack_trace = _extract_err_msg(episode_info)
+ else:
+ # useful until we get a proper place in agent_xray to view error
+ # messages.
+ if len(episode_info) == 0:
+ episode_info.append(StepInfo())
+ episode_info[-1].agent_info["err_msg"] = err_msg
+ episode_info[-1].agent_info["stack_trace"] = stack_trace
+
+ summary_info = dict(
+ n_steps=len(episode_info) - 1,
+ cum_reward=sum([step.reward for step in episode_info]),
+ cum_raw_reward=sum([step.raw_reward for step in episode_info if step.raw_reward]),
+ err_msg=err_msg,
+ stack_trace=stack_trace,
+ )
+ for key, val in _aggregate_episode_stats(episode_info).items():
+ summary_info[f"stats.{key}"] = val
+
+ if len(episode_info) > 0:
+ summary_info["terminated"] = episode_info[-1].terminated
+ summary_info["truncated"] = episode_info[-1].truncated
+
+ with open(exp_dir / "summary_info.json", "w") as f:
+ json.dump(summary_info, f, indent=4)
+
+
+def _extract_err_msg(episode_info: list[StepInfo]):
+ """Extract the last error message from the episode info."""
+ errors = [(None, None)]
+ for step_info in episode_info:
+ if step_info.agent_info is None:
+ continue
+ err_msg = step_info.agent_info.get("err_msg", None)
+ if err_msg is not None:
+ errors.append((err_msg, step_info.agent_info.get("stack_trace", None)))
+
+ return errors[-1]
+
+
+def _aggregate_episode_stats(episode_info: list[StepInfo]):
+ """Aggregate StepInfo.stats across episodes.
+
+ It will compute the sum and max of each value in the stats dict.
+ These two summaries should cover many use cases. If more are needed, the
+ user can compute other stats by reloading individual StepInfo.
+
+ Args:
+ episode_info: list[StepInfo]
+ The list of StepInfo objects to aggregate.
+
+ Returns:
+ dict
+ A dictionary containing the aggregated stats.
+ """
+
+ stats = defaultdict(list)
+ for step_info in episode_info:
+ if step_info.stats is not None:
+ for key, val in step_info.stats.items():
+ if val is None:
+ val = np.nan
+ stats[key].append(val)
+
+ aggregated_stats = {"cum_steps": len(episode_info)} # to be able to compute the mean
+ for key, val_list in stats.items():
+ aggregated_stats[f"cum_{key}"] = np.nansum(val_list)
+ aggregated_stats[f"max_{key}"] = np.nanmax(val_list)
+
+ for key, val in aggregated_stats.items():
+ if isinstance(val, np.generic):
+ aggregated_stats[key] = val.item()
+ if np.isnan(val):
+ aggregated_stats[key] = None
+ return aggregated_stats
+
+
+def _is_debugging():
+ """Tells you if your code is currently running in debug mode."""
+ return sys.gettrace() is not None
+
+
+class ExpResult:
+ """Helper class to load and visualize the results of an experiment.
+
+ attributes are loaded lazily.
+
+ Attributes (lazily loaded):
+ exp_args: ExpArgs, the arguments of the experiment.
+ steps_info: list[StepInfo], the information of each steps so far
+ summary_info: dict, the summary of the experiment.
+ screenshots: list[Image], the screenshots of each step.
+ screenshots_som: list[Image], the screenshots of each step with set of
+ marks inprinted.
+ flat_exp_args: dict, the flattened version of exp_args.
+ chat_video_path: Path, the path to the chat video. (if record_video=True)
+ task_video_path: Path, the path to the task video. (if record_video=True)
+ combined_video_path: Path, the path to the combined video. (if video was
+ combined)
+ """
+
+ def __init__(self, exp_dir) -> None:
+ self.exp_dir = Path(exp_dir)
+ self._exp_args = None
+ self._steps_info = {}
+ self._summary_info = None
+ self._screenshots = {}
+ self._flat_exp_args = None
+ self._logs = None
+
+ @property
+ def exp_args(self) -> ExpArgs:
+ if self._exp_args is None:
+ with open(self.exp_dir / "exp_args.pkl", "rb") as f:
+ self._exp_args = pickle.load(f)
+ # in case experiments were moved
+ self._exp_args.exp_dir = self.exp_dir
+ return self._exp_args
+
+ def get_step_info(self, step: int) -> StepInfo:
+ """Load the step info from the file and return it."""
+ if self._steps_info.get(step, None) is None:
+ with gzip.open(self.exp_dir / f"step_{step}.pkl.gz", "rb") as f:
+ self._steps_info[step] = pickle.load(f)
+ if self._steps_info[step].obs:
+ if "screenshot" not in self._steps_info[step].obs:
+ try:
+ self._steps_info[step].obs["screenshot"] = np.array(
+ self.get_screenshot(step), dtype=np.uint8
+ )
+ except FileNotFoundError:
+ pass
+ if "screenshot_som" not in self._steps_info[step].obs:
+ try:
+ self._steps_info[step].obs["screenshot_som"] = np.array(
+ self.get_screenshot(step, som=True), dtype=np.uint8
+ )
+ except FileNotFoundError:
+ pass
+ # if goal_object is set to None, it indicates it has been saved into a separate file
+ if (
+ "goal_object" in self._steps_info[step].obs
+ and self._steps_info[step].obs["goal_object"] is None
+ ):
+ with gzip.open(self.exp_dir / "goal_object.pkl.gz", "rb") as f:
+ goal_object = pickle.load(f)
+ self._steps_info[step].obs["goal_object"] = goal_object
+
+ return self._steps_info[step]
+
+ @property
+ def steps_info(self) -> list[StepInfo]:
+ step_files = list(self.exp_dir.glob("step_*.pkl.gz"))
+ for file in step_files:
+ step = int(file.name.split("_")[-1].split(".")[0])
+ self.get_step_info(step)
+
+ return [self._steps_info[i] for i in range(len(self._steps_info))]
+
+ @property
+ def summary_info(self) -> dict:
+ if self._summary_info is None:
+ with open(self.exp_dir / "summary_info.json", "r") as f:
+ # if length is zero raise file not found error
+ if os.fstat(f.fileno()).st_size == 0:
+ raise FileNotFoundError("summary_info.json is empty.")
+ self._summary_info = json.load(f)
+ return self._summary_info
+
+ def get_screenshot(self, step: int, som=False) -> Image:
+ key = (step, som)
+ if self._screenshots.get(key, None) is None:
+ file_name = f"screenshot_{'som_' if som else ''}step_{step}"
+ try:
+ with Image.open(self.exp_dir / (file_name + ".png")) as img:
+ self._screenshots[key] = img.copy()
+ except FileNotFoundError:
+ with Image.open(self.exp_dir / (file_name + ".jpg")) as img:
+ self._screenshots[key] = img.copy()
+ return self._screenshots[key]
+
+ def get_screenshots(self, som=False):
+ files = list(self.exp_dir.glob("screenshot_step_*"))
+ max_step = 0
+ for file in files:
+ step = int(file.name.split("_")[-1].split(".")[0])
+ self.get_screenshot(step, som=som)
+ max_step = max(max_step, step)
+ return [self._screenshots.get((i, som), None) for i in range(max_step + 1)]
+
+ @property
+ def screenshots(self):
+ return self.get_screenshots(som=False)
+
+ @property
+ def screenshots_som(self):
+ return self.get_screenshots(som=True)
+
+ @property
+ def flat_exp_args(self) -> dict:
+ """Return a dict with exp_args flattened."""
+ if self._flat_exp_args is None:
+ exp_args = asdict(self.exp_args)
+ # this will flatten nested dicts
+ self._flat_exp_args = _flatten_dict(exp_args)
+ return self._flat_exp_args
+
+ def get_exp_record(self) -> dict:
+ """Return a dict with exp_args flattened and summary_info."""
+ record = {"exp_dir": self.exp_dir}
+ try:
+ record.update(self.flat_exp_args)
+ except FileNotFoundError:
+ pass
+ try:
+ record.update(self.summary_info)
+ except FileNotFoundError:
+ pass
+ return record
+
+ @property
+ def chat_video_path(self) -> Path:
+ try:
+ return next(self.exp_dir.glob("chat_video/*.webm"))
+ except StopIteration:
+ raise FileNotFoundError(f"No chat_video found in {self.exp_dir}")
+
+ @property
+ def task_video_path(self) -> Path:
+ try:
+ return next(self.exp_dir.glob("task_video/*.webm"))
+ except StopIteration:
+ raise FileNotFoundError(f"No task_video found in {self.exp_dir}")
+
+ @property
+ def combined_video_path(self) -> Path:
+ return self.exp_dir / "combined_video.mp4"
+
+ @property
+ def logs(self):
+ if self._logs is None:
+ self._logs = (self.exp_dir / "experiment.log").read_text()
+ return self._logs
+
+ @property
+ def status(self):
+ """Possible values:
+ * "done": completed with no error
+ * "error": completed with error
+ * "incomplete": not completed yet (may be pending or just stalled)
+
+ Returns:
+ str: the status of the experiment. One of "done", "error", "incomplete".
+ """
+ try:
+ summary_info = self.summary_info
+ except FileNotFoundError:
+ return "incomplete"
+
+ if summary_info.get("err_msg", None) is not None:
+ return "error"
+
+ if summary_info.get("terminated", False) or summary_info.get("truncated", False):
+ return "done"
+
+ return "incomplete"
+
+
+EXP_RESULT_CACHE = {}
+
+
+def get_exp_result(exp_dir) -> ExpResult:
+ """Keep a cache of pre-loaded exp_results for faster loading"""
+ exp_dir = str(exp_dir) # make sure it's not a Path
+ exp_result = EXP_RESULT_CACHE.get(exp_dir, None)
+ if exp_result is None:
+ exp_result = ExpResult(exp_dir)
+ EXP_RESULT_CACHE[exp_dir] = exp_result
+ return exp_result
+
+
+def yield_all_exp_results(
+ savedir_base: str | Path, progress_fn=tqdm, load_hidden=False, use_cache=True
+):
+ """Recursively find all experiments from savedir_base folder.
+
+ This will ignore all experiments that start with "_" or ".". use
+ `load_hidden=True` to load them anyway.
+
+ Args:
+ savedir_base: str or Path
+ The base directory where the experiments are saved.
+ progress_fn: function
+ A function to show progress. Defaults to tqdm.
+ load_hidden: bool
+ If True, load hidden experiments (those starting with "_" or ".").
+ use_cache: bool
+ If True, use the cache of pre-loaded exp_results.
+
+ Yields:
+ ExpResult
+ An instance of ExpResult for each experiment found.
+ """
+
+ if not isinstance(savedir_base, list):
+ savedir_base = [savedir_base]
+
+ exp_args_paths = []
+ for exp_dir in savedir_base:
+ exp_args_paths.extend(list(Path(exp_dir).glob("**/exp_args.pkl")))
+
+ if progress_fn is not None:
+ exp_args_paths = progress_fn(exp_args_paths, desc="Searching experiments directories.")
+
+ for exp_args_path in exp_args_paths:
+ exp_dir = exp_args_path.parent
+ if not load_hidden:
+ if exp_dir.name.startswith("_") or exp_dir.name.startswith("."):
+ continue
+ if use_cache:
+ yield get_exp_result(exp_dir)
+ else:
+ yield ExpResult(exp_dir)
+
+
+class DataclassJSONEncoder(json.JSONEncoder):
+ def default(self, obj):
+ if is_dataclass(obj):
+ return asdict(obj)
+ if isinstance(obj, np.integer):
+ return int(obj)
+ if isinstance(obj, np.floating):
+ return float(obj)
+ if isinstance(obj, np.ndarray):
+ return obj.tolist()
+ return super().default(obj)
+
+
+def _move_old_exp(exp_dir):
+ """Move the old experiment directory to a new name."""
+ exp_dir = Path(exp_dir)
+ if exp_dir.exists():
+ exp_dir.rename(exp_dir.with_name("_" + exp_dir.name))
+
+
+def _get_env_name(task_name: str):
+ """Register tasks if needed (lazy import) and return environment name."""
+
+ # lazy benchmark import
+ if task_name.startswith("miniwob"):
+ import browsergym.miniwob
+ elif task_name.startswith("workarena"):
+ import browsergym.workarena
+ elif task_name.startswith("webarena"):
+ import browsergym.webarena
+ elif task_name.startswith("visualwebarena"):
+ import browsergym.visualwebarena
+ elif task_name.startswith("assistantbench"):
+ import browsergym.assistantbench
+ elif task_name.startswith("weblinx"):
+ import weblinx_browsergym
+
+ return f"browsergym/{task_name}"
+
+
+def _send_chat_info(chat: Chat, action: str, agent_info: dict):
+ """Send the think and action info to the chat."""
+ msg = ""
+ if "think" in agent_info:
+ msg += f"""\
+{agent_info["think"]}
+
+"""
+
+ msg += f"""\
+action:
+{action}
+"""
+
+ logger.info(msg)
+ chat.add_message(role="info", msg=msg)
+
+
+def _flatten_dict(d, parent_key="", sep="."):
+ """Recursively flatten a nested dictionary."""
+ items = []
+ for k, v in d.items():
+ new_key = parent_key + sep + k if parent_key else k
+ if isinstance(v, dict):
+ items.extend(_flatten_dict(v, new_key, sep).items())
+ else:
+ items.append((new_key, v))
+ return dict(items)
diff --git a/src/agentlab/experiments/study.py b/src/agentlab/experiments/study.py
index b93b3ae2..7de3db98 100644
--- a/src/agentlab/experiments/study.py
+++ b/src/agentlab/experiments/study.py
@@ -12,15 +12,20 @@
from pathlib import Path
import bgym
-from bgym import Benchmark, EnvArgs, ExpArgs
+from bgym import Benchmark
from slugify import slugify
from agentlab.agents.agent_args import AgentArgs
from agentlab.analyze import inspect_results
from agentlab.experiments import reproducibility_util as repro
from agentlab.experiments.exp_utils import RESULTS_DIR, add_dependencies
-from agentlab.experiments.launch_exp import find_incomplete, non_dummy_count, run_experiments
-from agentlab.experiments.multi_server import BaseServer, WebArenaInstanceVars
+from agentlab.experiments.launch_exp import (
+ find_incomplete,
+ non_dummy_count,
+ run_experiments,
+)
+from agentlab.experiments.loop import EnvArgs, ExpArgs
+from agentlab.experiments.multi_server import BaseServer
logger = logging.getLogger(__name__)
@@ -28,6 +33,7 @@
def make_study(
agent_args: list[AgentArgs] | AgentArgs,
benchmark: bgym.Benchmark | str,
+ logging_level=logging.WARNING,
logging_level_stdout=logging.WARNING,
suffix="",
comment=None,
@@ -44,6 +50,8 @@ def make_study(
benchmark: bgym.Benchmark | str
The benchmark to run the agents on. See bgym.DEFAULT_BENCHMARKS for the main ones. You
can also make your own by modifying an existing one.
+ logging_level: int
+ The logging level for file log.
logging_level_stdout: int
The logging level for the stdout of the main script. Each job will have its own logging
level that will save into file and can be seen in agentlab-xray.
@@ -93,7 +101,8 @@ def make_study(
Study(
[agent],
benchmark,
- logging_level=logging_level_stdout,
+ logging_level=logging_level,
+ logging_level_stdout=logging_level_stdout,
suffix=suffix,
comment=comment,
ignore_dependencies=ignore_dependencies,
@@ -107,7 +116,8 @@ def make_study(
return Study(
agent_args,
benchmark,
- logging_level=logging_level_stdout,
+ logging_level=logging_level,
+ logging_level_stdout=logging_level_stdout,
suffix=suffix,
comment=comment,
ignore_dependencies=ignore_dependencies,
@@ -305,7 +315,6 @@ def run(
relaunch_errors=True,
exp_root=RESULTS_DIR,
):
-
self.set_reproducibility_info(
strict_reproducibility=strict_reproducibility, comment=self.comment
)
@@ -325,12 +334,12 @@ def run(
n_incomplete, n_error = self.find_incomplete(include_errors=relaunch_errors)
if n_error / n_exp > 0.3:
- logger.warning(f"More than 30% of the experiments errored. Stopping the study.")
+ logger.warning("More than 30% of the experiments errored. Stopping the study.")
return
if last_error_count is not None and n_error >= last_error_count:
logger.warning(
- f"Last trial did not reduce the number of errors. Stopping the study."
+ "Last trial did not reduce the number of errors. Stopping the study."
)
return
@@ -553,7 +562,6 @@ def run(
n_relaunch=3,
exp_root=RESULTS_DIR,
):
-
# This sequence of of making directories is important to make sure objects are materialized
# properly before saving. Otherwise relaunch may not work properly.
self.make_dir()
@@ -640,7 +648,6 @@ def _run(
@dataclass
class ParallelStudies_alt(SequentialStudies):
-
parallel_servers: list[BaseServer] | int = None
def _run(
diff --git a/src/agentlab/ui_assistant.py b/src/agentlab/ui_assistant.py
index 96bbb0f9..57916543 100644
--- a/src/agentlab/ui_assistant.py
+++ b/src/agentlab/ui_assistant.py
@@ -1,15 +1,13 @@
import argparse
-from browsergym.experiments.loop import EnvArgs, ExpArgs
-
from agentlab.agents.agent_args import AgentArgs
from agentlab.agents.generic_agent.generic_agent import GenericAgentArgs
from agentlab.experiments.exp_utils import RESULTS_DIR
from agentlab.experiments.launch_exp import import_object
+from agentlab.experiments.loop import EnvArgs, ExpArgs
def make_exp_args(agent_args: AgentArgs, start_url="https://www.google.com"):
-
try:
agent_args.flags.action.demo_mode = "default"
except AttributeError:
diff --git a/tests/agents/test_agent.py b/tests/agents/test_agent.py
index 2632f66b..7fcaafa2 100644
--- a/tests/agents/test_agent.py
+++ b/tests/agents/test_agent.py
@@ -3,13 +3,13 @@
from dataclasses import dataclass
from pathlib import Path
-from browsergym.experiments.loop import EnvArgs, ExpArgs
from openai import OpenAIError
from agentlab.agents.generic_agent.agent_configs import FLAGS_GPT_3_5
from agentlab.agents.generic_agent.generic_agent import GenericAgentArgs
from agentlab.analyze import inspect_results
from agentlab.experiments import launch_exp
+from agentlab.experiments.loop import EnvArgs, ExpArgs
from agentlab.llm.chat_api import BaseModelArgs, CheatMiniWoBLLMArgs
from agentlab.llm.llm_utils import Discussion
@@ -24,7 +24,6 @@ def test_generic_agent():
)
with tempfile.TemporaryDirectory() as tmp_dir:
-
launch_exp.run_experiments(
1, [exp_args], Path(tmp_dir) / "generic_agent_test", parallel_backend="joblib"
)
diff --git a/tests/agents/test_gaia_agent.py b/tests/agents/test_gaia_agent.py
new file mode 100644
index 00000000..0d39f9ef
--- /dev/null
+++ b/tests/agents/test_gaia_agent.py
@@ -0,0 +1,97 @@
+import os
+import uuid
+from pathlib import Path
+
+from tapeagents.steps import ImageObservation
+
+from agentlab.agents.tapeagent.agent import TapeAgent, TapeAgentArgs, load_config
+from agentlab.benchmarks.gaia import GaiaBenchmark, GaiaQuestion
+
+
+def mock_dataset() -> dict:
+ """Mock dataset for testing purposes."""
+ data = [{"task_id": str(uuid.uuid4()), "file_name": "", "file_path": ""} for i in range(165)]
+ data[5] = {
+ "task_id": "32102e3e-d12a-4209-9163-7b3a104efe5d",
+ "Question": """The attached spreadsheet shows the inventory for a movie and video game rental store in Seattle, Washington. What is the title of the oldest Blu-Ray recorded in this spreadsheet? Return it as appearing in the spreadsheet.""",
+ "Level": "2",
+ "Final answer": "Time-Parking 2: Parallel Universe",
+ "file_name": "32102e3e-d12a-4209-9163-7b3a104efe5d.xlsx",
+ "file_path": "tests/data/32102e3e-d12a-4209-9163-7b3a104efe5d.xlsx",
+ "Annotator Metadata": {
+ "Steps": """1. Open the attached file.\n2. Compare the years given in the Blu-Ray section to find the oldest year, 2009.\n3. Find the title of the Blu-Ray disc that corresponds to the year 2009: Time-Parking 2: Parallel Universe.""",
+ "Number of steps": "3",
+ "How long did this take?": "1 minute",
+ "Tools": "1. Microsoft Excel",
+ "Number of tools": "1",
+ },
+ }
+ data[20] = {
+ "task_id": "df6561b2-7ee5-4540-baab-5095f742716a",
+ "Question": "When you take the average of the standard population deviation of the red numbers and the standard sample deviation of the green numbers in this image using the statistics module in Python 3.11, what is the result rounded to the nearest three decimal points?",
+ "Level": "2",
+ "Final answer": "17.056",
+ "file_name": "df6561b2-7ee5-4540-baab-5095f742716a.png",
+ "file_path": "tests/data/df6561b2-7ee5-4540-baab-5095f742716a.png",
+ "Annotator Metadata": {
+ "Steps": "1. Opened the PNG file.\n2. Made separate lists of the red numbers and green numbers.\n3. Opened a Python compiler.\n4. Ran the following code:\n```\nimport statistics as st\nred = st.pstdev([24, 74, 28, 54, 73, 33, 64, 73, 60, 53, 59, 40, 65, 76, 48, 34, 62, 70, 31, 24, 51, 55, 78, 76, 41, 77, 51])\ngreen = st.stdev([39, 29, 28, 72, 68, 47, 64, 74, 72, 40, 75, 26, 27, 37, 31, 55, 44, 64, 65, 38, 46, 66, 35, 76, 61, 53, 49])\navg = st.mean([red, green])\nprint(avg)\n```\n5. Rounded the output.",
+ "Number of steps": "5",
+ "How long did this take?": "20 minutes",
+ "Tools": "1. Python compiler\n2. Image recognition tools",
+ "Number of tools": "2",
+ },
+ }
+ return {"validation": data}
+
+
+def test_agent_creation():
+ config = load_config("gaia_val")
+ args = TapeAgentArgs(config=config)
+ agent = args.make_agent()
+ assert isinstance(agent, TapeAgent)
+ assert agent.agent.name == "gaia_agent"
+
+
+def test_gaia_bench():
+ config = load_config("gaia_val")
+ bench = GaiaBenchmark.from_config(config, dataset=mock_dataset())
+ assert bench.name == "gaia"
+ assert bench.split == "validation"
+ assert len(bench.env_args_list) == 165
+
+ task = bench.env_args_list[5].task
+ question = """The attached spreadsheet shows the inventory for a movie and video game rental store in Seattle, Washington. What is the title of the oldest Blu-Ray recorded in this spreadsheet? Return it as appearing in the spreadsheet."""
+ steps = """1. Open the attached file.\n2. Compare the years given in the Blu-Ray section to find the oldest year, 2009.\n3. Find the title of the Blu-Ray disc that corresponds to the year 2009: Time-Parking 2: Parallel Universe."""
+ assert task["task_id"] == "32102e3e-d12a-4209-9163-7b3a104efe5d"
+ assert task["Question"] == question
+ assert task["Level"] == "2"
+ assert task["Final answer"] == "Time-Parking 2: Parallel Universe"
+ assert task["file_name"] == "32102e3e-d12a-4209-9163-7b3a104efe5d.xlsx"
+ assert task["Annotator Metadata"]["Steps"] == steps
+ assert task["Annotator Metadata"]["Number of steps"] == "3"
+ assert task["Annotator Metadata"]["How long did this take?"] == "1 minute"
+ assert task["Annotator Metadata"]["Tools"] == "1. Microsoft Excel"
+ assert task["Annotator Metadata"]["Number of tools"] == "1"
+
+
+def test_gaia_gym_reset():
+ exp_dir = "/tmp/gaia_unit_test"
+ os.makedirs(exp_dir, exist_ok=True)
+
+ config = load_config("gaia_val")
+ bench = GaiaBenchmark.from_config(config, dataset=mock_dataset())
+ args = bench.env_args_list[5]
+ env = args.make_env(Path(exp_dir))
+ steps, _ = env.reset()
+ assert len(steps) == 1
+ assert isinstance(steps[0], GaiaQuestion)
+ assert steps[0].content.startswith(args.task["Question"])
+
+ args = bench.env_args_list[20]
+ env = args.make_env(Path(exp_dir))
+ steps, _ = env.reset()
+ assert len(steps) == 2
+ assert isinstance(steps[0], GaiaQuestion)
+ assert steps[0].content == args.task["Question"]
+ assert isinstance(steps[1], ImageObservation)
+ assert os.path.basename(steps[1].image_path) == args.task["file_name"]
diff --git a/tests/agents/test_visualwebarena_agent.py b/tests/agents/test_visualwebarena_agent.py
index 33547a77..4cdd6fb6 100644
--- a/tests/agents/test_visualwebarena_agent.py
+++ b/tests/agents/test_visualwebarena_agent.py
@@ -2,9 +2,9 @@
import tempfile
import pytest
-from browsergym.experiments.loop import EnvArgs, ExpArgs
from agentlab.agents.visualwebarena.agent import VisualWebArenaAgentArgs
+from agentlab.experiments.loop import EnvArgs, ExpArgs
from agentlab.llm.llm_configs import CHAT_MODEL_ARGS_DICT
diff --git a/tests/data/32102e3e-d12a-4209-9163-7b3a104efe5d.xlsx b/tests/data/32102e3e-d12a-4209-9163-7b3a104efe5d.xlsx
new file mode 100644
index 00000000..c8bd42f2
Binary files /dev/null and b/tests/data/32102e3e-d12a-4209-9163-7b3a104efe5d.xlsx differ
diff --git a/tests/data/df6561b2-7ee5-4540-baab-5095f742716a.png b/tests/data/df6561b2-7ee5-4540-baab-5095f742716a.png
new file mode 100644
index 00000000..88e81bee
Binary files /dev/null and b/tests/data/df6561b2-7ee5-4540-baab-5095f742716a.png differ
diff --git a/tests/experiments/test_launch_exp.py b/tests/experiments/test_launch_exp.py
index 1a58f797..384118bd 100644
--- a/tests/experiments/test_launch_exp.py
+++ b/tests/experiments/test_launch_exp.py
@@ -3,12 +3,16 @@
from pathlib import Path
import pytest
-from browsergym.experiments.loop import EnvArgs, ExpArgs
from agentlab.agents.generic_agent.agent_configs import FLAGS_GPT_3_5, AGENT_4o_MINI
from agentlab.agents.generic_agent.generic_agent import GenericAgentArgs
from agentlab.analyze import inspect_results
-from agentlab.experiments.launch_exp import find_incomplete, non_dummy_count, run_experiments
+from agentlab.experiments.launch_exp import (
+ find_incomplete,
+ non_dummy_count,
+ run_experiments,
+)
+from agentlab.experiments.loop import EnvArgs, ExpArgs
from agentlab.experiments.study import Study
from agentlab.llm.chat_api import CheatMiniWoBLLMArgs
@@ -26,7 +30,6 @@ def test_relaunch_study():
def _test_launch_system(backend="ray", cause_timeout=False):
-
if cause_timeout:
wait_time = 10
avg_step_timeout = 0.5
@@ -47,7 +50,6 @@ def _test_launch_system(backend="ray", cause_timeout=False):
)
with tempfile.TemporaryDirectory() as tmp_dir:
-
study_dir = Path(tmp_dir) / "generic_agent_test"
run_experiments(
n_jobs=2,
@@ -100,7 +102,6 @@ def test_timeout_ray():
def test_4o_mini_on_miniwob_tiny_test():
"""Run with `pytest -m pricy`."""
with tempfile.TemporaryDirectory() as tmp_dir:
-
study = Study(agent_args=[AGENT_4o_MINI], benchmark="miniwob_tiny_test", dir=tmp_dir)
study.run(n_jobs=4)
diff --git a/tests/experiments/test_ray.py b/tests/experiments/test_ray.py
index 28ddfa34..acab7b73 100644
--- a/tests/experiments/test_ray.py
+++ b/tests/experiments/test_ray.py
@@ -32,15 +32,16 @@ def test_execute_task_graph():
assert exp_args_list[2].end_time < exp_args_list[3].start_time
# Verify that parallel tasks (task2 and task3) started within a short time of each other
- parallel_start_diff = abs(exp_args_list[1].start_time - exp_args_list[2].start_time)
- print(f"parallel_start_diff: {parallel_start_diff}")
- assert parallel_start_diff < 2 # Allow for a small delay
+ # TODO: replace with non flaky check
+ # parallel_start_diff = abs(exp_args_list[1].start_time - exp_args_list[2].start_time)
+ # print(f"parallel_start_diff: {parallel_start_diff}")
+ # assert parallel_start_diff < 2, "Parallel tasks should start within 2 seconds of each other"
# Ensure that the entire task graph took the expected amount of time
- total_time = exp_args_list[-1].end_time - exp_args_list[0].start_time
- assert (
- total_time >= TASK_TIME * 3
- ) # Since the critical path involves at least 1.5 seconds of work
+ # TODO: replace with non flaky check
+ # total_time = exp_args_list[-1].end_time - exp_args_list[0].start_time
+ # # Since the critical path involves at least 1.5 seconds of work
+ # assert total_time >= TASK_TIME * 3, "Total time should be at least 3 times the task time"
def test_add_dependencies():